src/cpu/x86/vm/x86_64.ad

Wed, 13 Oct 2010 11:46:46 -0400

author
acorn
date
Wed, 13 Oct 2010 11:46:46 -0400
changeset 2226
75b0735b4d04
parent 2182
52e82a6bedaf
child 2269
ae065c367d93
permissions
-rw-r--r--

Merge

     1 //
     2 // Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
     3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     4 //
     5 // This code is free software; you can redistribute it and/or modify it
     6 // under the terms of the GNU General Public License version 2 only, as
     7 // published by the Free Software Foundation.
     8 //
     9 // This code is distributed in the hope that it will be useful, but WITHOUT
    10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    12 // version 2 for more details (a copy is included in the LICENSE file that
    13 // accompanied this code).
    14 //
    15 // You should have received a copy of the GNU General Public License version
    16 // 2 along with this work; if not, write to the Free Software Foundation,
    17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
    18 //
    19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
    20 // or visit www.oracle.com if you need additional information or have any
    21 // questions.
    22 //
    23 //
    25 // AMD64 Architecture Description File
    27 //----------REGISTER DEFINITION BLOCK------------------------------------------
    28 // This information is used by the matcher and the register allocator to
    29 // describe individual registers and classes of registers within the target
    30 // archtecture.
    32 register %{
    33 //----------Architecture Description Register Definitions----------------------
    34 // General Registers
    35 // "reg_def"  name ( register save type, C convention save type,
    36 //                   ideal register type, encoding );
    37 // Register Save Types:
    38 //
    39 // NS  = No-Save:       The register allocator assumes that these registers
    40 //                      can be used without saving upon entry to the method, &
    41 //                      that they do not need to be saved at call sites.
    42 //
    43 // SOC = Save-On-Call:  The register allocator assumes that these registers
    44 //                      can be used without saving upon entry to the method,
    45 //                      but that they must be saved at call sites.
    46 //
    47 // SOE = Save-On-Entry: The register allocator assumes that these registers
    48 //                      must be saved before using them upon entry to the
    49 //                      method, but they do not need to be saved at call
    50 //                      sites.
    51 //
    52 // AS  = Always-Save:   The register allocator assumes that these registers
    53 //                      must be saved before using them upon entry to the
    54 //                      method, & that they must be saved at call sites.
    55 //
    56 // Ideal Register Type is used to determine how to save & restore a
    57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
    58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
    59 //
    60 // The encoding number is the actual bit-pattern placed into the opcodes.
    62 // General Registers
    63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
    64 // used as byte registers)
    66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
    67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
    68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
    70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
    71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
    73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
    74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
    76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
    77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
    79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
    80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
    82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
    83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
    85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
    86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
    87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
    89 #ifdef _WIN64
    91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
    92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
    94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
    95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
    97 #else
    99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
   100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
   102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
   103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
   105 #endif
   107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
   108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
   110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
   111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
   113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
   114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
   116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
   117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
   119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
   120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
   122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
   123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
   125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
   126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
   128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
   129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
   132 // Floating Point Registers
   134 // XMM registers.  128-bit registers or 4 words each, labeled (a)-d.
   135 // Word a in each register holds a Float, words ab hold a Double.  We
   136 // currently do not use the SIMD capabilities, so registers cd are
   137 // unused at the moment.
   138 // XMM8-XMM15 must be encoded with REX.
   139 // Linux ABI:   No register preserved across function calls
   140 //              XMM0-XMM7 might hold parameters
   141 // Windows ABI: XMM6-XMM15 preserved across function calls
   142 //              XMM0-XMM3 might hold parameters
   144 reg_def XMM0   (SOC, SOC, Op_RegF,  0, xmm0->as_VMReg());
   145 reg_def XMM0_H (SOC, SOC, Op_RegF,  0, xmm0->as_VMReg()->next());
   147 reg_def XMM1   (SOC, SOC, Op_RegF,  1, xmm1->as_VMReg());
   148 reg_def XMM1_H (SOC, SOC, Op_RegF,  1, xmm1->as_VMReg()->next());
   150 reg_def XMM2   (SOC, SOC, Op_RegF,  2, xmm2->as_VMReg());
   151 reg_def XMM2_H (SOC, SOC, Op_RegF,  2, xmm2->as_VMReg()->next());
   153 reg_def XMM3   (SOC, SOC, Op_RegF,  3, xmm3->as_VMReg());
   154 reg_def XMM3_H (SOC, SOC, Op_RegF,  3, xmm3->as_VMReg()->next());
   156 reg_def XMM4   (SOC, SOC, Op_RegF,  4, xmm4->as_VMReg());
   157 reg_def XMM4_H (SOC, SOC, Op_RegF,  4, xmm4->as_VMReg()->next());
   159 reg_def XMM5   (SOC, SOC, Op_RegF,  5, xmm5->as_VMReg());
   160 reg_def XMM5_H (SOC, SOC, Op_RegF,  5, xmm5->as_VMReg()->next());
   162 #ifdef _WIN64
   164 reg_def XMM6   (SOC, SOE, Op_RegF,  6, xmm6->as_VMReg());
   165 reg_def XMM6_H (SOC, SOE, Op_RegF,  6, xmm6->as_VMReg()->next());
   167 reg_def XMM7   (SOC, SOE, Op_RegF,  7, xmm7->as_VMReg());
   168 reg_def XMM7_H (SOC, SOE, Op_RegF,  7, xmm7->as_VMReg()->next());
   170 reg_def XMM8   (SOC, SOE, Op_RegF,  8, xmm8->as_VMReg());
   171 reg_def XMM8_H (SOC, SOE, Op_RegF,  8, xmm8->as_VMReg()->next());
   173 reg_def XMM9   (SOC, SOE, Op_RegF,  9, xmm9->as_VMReg());
   174 reg_def XMM9_H (SOC, SOE, Op_RegF,  9, xmm9->as_VMReg()->next());
   176 reg_def XMM10  (SOC, SOE, Op_RegF, 10, xmm10->as_VMReg());
   177 reg_def XMM10_H(SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next());
   179 reg_def XMM11  (SOC, SOE, Op_RegF, 11, xmm11->as_VMReg());
   180 reg_def XMM11_H(SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next());
   182 reg_def XMM12  (SOC, SOE, Op_RegF, 12, xmm12->as_VMReg());
   183 reg_def XMM12_H(SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next());
   185 reg_def XMM13  (SOC, SOE, Op_RegF, 13, xmm13->as_VMReg());
   186 reg_def XMM13_H(SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next());
   188 reg_def XMM14  (SOC, SOE, Op_RegF, 14, xmm14->as_VMReg());
   189 reg_def XMM14_H(SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next());
   191 reg_def XMM15  (SOC, SOE, Op_RegF, 15, xmm15->as_VMReg());
   192 reg_def XMM15_H(SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next());
   194 #else
   196 reg_def XMM6   (SOC, SOC, Op_RegF,  6, xmm6->as_VMReg());
   197 reg_def XMM6_H (SOC, SOC, Op_RegF,  6, xmm6->as_VMReg()->next());
   199 reg_def XMM7   (SOC, SOC, Op_RegF,  7, xmm7->as_VMReg());
   200 reg_def XMM7_H (SOC, SOC, Op_RegF,  7, xmm7->as_VMReg()->next());
   202 reg_def XMM8   (SOC, SOC, Op_RegF,  8, xmm8->as_VMReg());
   203 reg_def XMM8_H (SOC, SOC, Op_RegF,  8, xmm8->as_VMReg()->next());
   205 reg_def XMM9   (SOC, SOC, Op_RegF,  9, xmm9->as_VMReg());
   206 reg_def XMM9_H (SOC, SOC, Op_RegF,  9, xmm9->as_VMReg()->next());
   208 reg_def XMM10  (SOC, SOC, Op_RegF, 10, xmm10->as_VMReg());
   209 reg_def XMM10_H(SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next());
   211 reg_def XMM11  (SOC, SOC, Op_RegF, 11, xmm11->as_VMReg());
   212 reg_def XMM11_H(SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next());
   214 reg_def XMM12  (SOC, SOC, Op_RegF, 12, xmm12->as_VMReg());
   215 reg_def XMM12_H(SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next());
   217 reg_def XMM13  (SOC, SOC, Op_RegF, 13, xmm13->as_VMReg());
   218 reg_def XMM13_H(SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next());
   220 reg_def XMM14  (SOC, SOC, Op_RegF, 14, xmm14->as_VMReg());
   221 reg_def XMM14_H(SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next());
   223 reg_def XMM15  (SOC, SOC, Op_RegF, 15, xmm15->as_VMReg());
   224 reg_def XMM15_H(SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next());
   226 #endif // _WIN64
   228 reg_def RFLAGS(SOC, SOC, 0, 16, VMRegImpl::Bad());
   230 // Specify priority of register selection within phases of register
   231 // allocation.  Highest priority is first.  A useful heuristic is to
   232 // give registers a low priority when they are required by machine
   233 // instructions, like EAX and EDX on I486, and choose no-save registers
   234 // before save-on-call, & save-on-call before save-on-entry.  Registers
   235 // which participate in fixed calling sequences should come last.
   236 // Registers which are used as pairs must fall on an even boundary.
   238 alloc_class chunk0(R10,         R10_H,
   239                    R11,         R11_H,
   240                    R8,          R8_H,
   241                    R9,          R9_H,
   242                    R12,         R12_H,
   243                    RCX,         RCX_H,
   244                    RBX,         RBX_H,
   245                    RDI,         RDI_H,
   246                    RDX,         RDX_H,
   247                    RSI,         RSI_H,
   248                    RAX,         RAX_H,
   249                    RBP,         RBP_H,
   250                    R13,         R13_H,
   251                    R14,         R14_H,
   252                    R15,         R15_H,
   253                    RSP,         RSP_H);
   255 // XXX probably use 8-15 first on Linux
   256 alloc_class chunk1(XMM0,  XMM0_H,
   257                    XMM1,  XMM1_H,
   258                    XMM2,  XMM2_H,
   259                    XMM3,  XMM3_H,
   260                    XMM4,  XMM4_H,
   261                    XMM5,  XMM5_H,
   262                    XMM6,  XMM6_H,
   263                    XMM7,  XMM7_H,
   264                    XMM8,  XMM8_H,
   265                    XMM9,  XMM9_H,
   266                    XMM10, XMM10_H,
   267                    XMM11, XMM11_H,
   268                    XMM12, XMM12_H,
   269                    XMM13, XMM13_H,
   270                    XMM14, XMM14_H,
   271                    XMM15, XMM15_H);
   273 alloc_class chunk2(RFLAGS);
   276 //----------Architecture Description Register Classes--------------------------
   277 // Several register classes are automatically defined based upon information in
   278 // this architecture description.
   279 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
   280 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
   281 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
   282 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
   283 //
   285 // Class for all pointer registers (including RSP)
   286 reg_class any_reg(RAX, RAX_H,
   287                   RDX, RDX_H,
   288                   RBP, RBP_H,
   289                   RDI, RDI_H,
   290                   RSI, RSI_H,
   291                   RCX, RCX_H,
   292                   RBX, RBX_H,
   293                   RSP, RSP_H,
   294                   R8,  R8_H,
   295                   R9,  R9_H,
   296                   R10, R10_H,
   297                   R11, R11_H,
   298                   R12, R12_H,
   299                   R13, R13_H,
   300                   R14, R14_H,
   301                   R15, R15_H);
   303 // Class for all pointer registers except RSP
   304 reg_class ptr_reg(RAX, RAX_H,
   305                   RDX, RDX_H,
   306                   RBP, RBP_H,
   307                   RDI, RDI_H,
   308                   RSI, RSI_H,
   309                   RCX, RCX_H,
   310                   RBX, RBX_H,
   311                   R8,  R8_H,
   312                   R9,  R9_H,
   313                   R10, R10_H,
   314                   R11, R11_H,
   315                   R13, R13_H,
   316                   R14, R14_H);
   318 // Class for all pointer registers except RAX and RSP
   319 reg_class ptr_no_rax_reg(RDX, RDX_H,
   320                          RBP, RBP_H,
   321                          RDI, RDI_H,
   322                          RSI, RSI_H,
   323                          RCX, RCX_H,
   324                          RBX, RBX_H,
   325                          R8,  R8_H,
   326                          R9,  R9_H,
   327                          R10, R10_H,
   328                          R11, R11_H,
   329                          R13, R13_H,
   330                          R14, R14_H);
   332 reg_class ptr_no_rbp_reg(RDX, RDX_H,
   333                          RAX, RAX_H,
   334                          RDI, RDI_H,
   335                          RSI, RSI_H,
   336                          RCX, RCX_H,
   337                          RBX, RBX_H,
   338                          R8,  R8_H,
   339                          R9,  R9_H,
   340                          R10, R10_H,
   341                          R11, R11_H,
   342                          R13, R13_H,
   343                          R14, R14_H);
   345 // Class for all pointer registers except RAX, RBX and RSP
   346 reg_class ptr_no_rax_rbx_reg(RDX, RDX_H,
   347                              RBP, RBP_H,
   348                              RDI, RDI_H,
   349                              RSI, RSI_H,
   350                              RCX, RCX_H,
   351                              R8,  R8_H,
   352                              R9,  R9_H,
   353                              R10, R10_H,
   354                              R11, R11_H,
   355                              R13, R13_H,
   356                              R14, R14_H);
   358 // Singleton class for RAX pointer register
   359 reg_class ptr_rax_reg(RAX, RAX_H);
   361 // Singleton class for RBX pointer register
   362 reg_class ptr_rbx_reg(RBX, RBX_H);
   364 // Singleton class for RSI pointer register
   365 reg_class ptr_rsi_reg(RSI, RSI_H);
   367 // Singleton class for RDI pointer register
   368 reg_class ptr_rdi_reg(RDI, RDI_H);
   370 // Singleton class for RBP pointer register
   371 reg_class ptr_rbp_reg(RBP, RBP_H);
   373 // Singleton class for stack pointer
   374 reg_class ptr_rsp_reg(RSP, RSP_H);
   376 // Singleton class for TLS pointer
   377 reg_class ptr_r15_reg(R15, R15_H);
   379 // Class for all long registers (except RSP)
   380 reg_class long_reg(RAX, RAX_H,
   381                    RDX, RDX_H,
   382                    RBP, RBP_H,
   383                    RDI, RDI_H,
   384                    RSI, RSI_H,
   385                    RCX, RCX_H,
   386                    RBX, RBX_H,
   387                    R8,  R8_H,
   388                    R9,  R9_H,
   389                    R10, R10_H,
   390                    R11, R11_H,
   391                    R13, R13_H,
   392                    R14, R14_H);
   394 // Class for all long registers except RAX, RDX (and RSP)
   395 reg_class long_no_rax_rdx_reg(RBP, RBP_H,
   396                               RDI, RDI_H,
   397                               RSI, RSI_H,
   398                               RCX, RCX_H,
   399                               RBX, RBX_H,
   400                               R8,  R8_H,
   401                               R9,  R9_H,
   402                               R10, R10_H,
   403                               R11, R11_H,
   404                               R13, R13_H,
   405                               R14, R14_H);
   407 // Class for all long registers except RCX (and RSP)
   408 reg_class long_no_rcx_reg(RBP, RBP_H,
   409                           RDI, RDI_H,
   410                           RSI, RSI_H,
   411                           RAX, RAX_H,
   412                           RDX, RDX_H,
   413                           RBX, RBX_H,
   414                           R8,  R8_H,
   415                           R9,  R9_H,
   416                           R10, R10_H,
   417                           R11, R11_H,
   418                           R13, R13_H,
   419                           R14, R14_H);
   421 // Class for all long registers except RAX (and RSP)
   422 reg_class long_no_rax_reg(RBP, RBP_H,
   423                           RDX, RDX_H,
   424                           RDI, RDI_H,
   425                           RSI, RSI_H,
   426                           RCX, RCX_H,
   427                           RBX, RBX_H,
   428                           R8,  R8_H,
   429                           R9,  R9_H,
   430                           R10, R10_H,
   431                           R11, R11_H,
   432                           R13, R13_H,
   433                           R14, R14_H);
   435 // Singleton class for RAX long register
   436 reg_class long_rax_reg(RAX, RAX_H);
   438 // Singleton class for RCX long register
   439 reg_class long_rcx_reg(RCX, RCX_H);
   441 // Singleton class for RDX long register
   442 reg_class long_rdx_reg(RDX, RDX_H);
   444 // Class for all int registers (except RSP)
   445 reg_class int_reg(RAX,
   446                   RDX,
   447                   RBP,
   448                   RDI,
   449                   RSI,
   450                   RCX,
   451                   RBX,
   452                   R8,
   453                   R9,
   454                   R10,
   455                   R11,
   456                   R13,
   457                   R14);
   459 // Class for all int registers except RCX (and RSP)
   460 reg_class int_no_rcx_reg(RAX,
   461                          RDX,
   462                          RBP,
   463                          RDI,
   464                          RSI,
   465                          RBX,
   466                          R8,
   467                          R9,
   468                          R10,
   469                          R11,
   470                          R13,
   471                          R14);
   473 // Class for all int registers except RAX, RDX (and RSP)
   474 reg_class int_no_rax_rdx_reg(RBP,
   475                              RDI,
   476                              RSI,
   477                              RCX,
   478                              RBX,
   479                              R8,
   480                              R9,
   481                              R10,
   482                              R11,
   483                              R13,
   484                              R14);
   486 // Singleton class for RAX int register
   487 reg_class int_rax_reg(RAX);
   489 // Singleton class for RBX int register
   490 reg_class int_rbx_reg(RBX);
   492 // Singleton class for RCX int register
   493 reg_class int_rcx_reg(RCX);
   495 // Singleton class for RCX int register
   496 reg_class int_rdx_reg(RDX);
   498 // Singleton class for RCX int register
   499 reg_class int_rdi_reg(RDI);
   501 // Singleton class for instruction pointer
   502 // reg_class ip_reg(RIP);
   504 // Singleton class for condition codes
   505 reg_class int_flags(RFLAGS);
   507 // Class for all float registers
   508 reg_class float_reg(XMM0,
   509                     XMM1,
   510                     XMM2,
   511                     XMM3,
   512                     XMM4,
   513                     XMM5,
   514                     XMM6,
   515                     XMM7,
   516                     XMM8,
   517                     XMM9,
   518                     XMM10,
   519                     XMM11,
   520                     XMM12,
   521                     XMM13,
   522                     XMM14,
   523                     XMM15);
   525 // Class for all double registers
   526 reg_class double_reg(XMM0,  XMM0_H,
   527                      XMM1,  XMM1_H,
   528                      XMM2,  XMM2_H,
   529                      XMM3,  XMM3_H,
   530                      XMM4,  XMM4_H,
   531                      XMM5,  XMM5_H,
   532                      XMM6,  XMM6_H,
   533                      XMM7,  XMM7_H,
   534                      XMM8,  XMM8_H,
   535                      XMM9,  XMM9_H,
   536                      XMM10, XMM10_H,
   537                      XMM11, XMM11_H,
   538                      XMM12, XMM12_H,
   539                      XMM13, XMM13_H,
   540                      XMM14, XMM14_H,
   541                      XMM15, XMM15_H);
   542 %}
   545 //----------SOURCE BLOCK-------------------------------------------------------
   546 // This is a block of C++ code which provides values, functions, and
   547 // definitions necessary in the rest of the architecture description
   548 source %{
   549 #define   RELOC_IMM64    Assembler::imm_operand
   550 #define   RELOC_DISP32   Assembler::disp32_operand
   552 #define __ _masm.
   554 static int preserve_SP_size() {
   555   return LP64_ONLY(1 +) 2;  // [rex,] op, rm(reg/reg)
   556 }
   558 // !!!!! Special hack to get all types of calls to specify the byte offset
   559 //       from the start of the call to the point where the return address
   560 //       will point.
   561 int MachCallStaticJavaNode::ret_addr_offset()
   562 {
   563   int offset = 5; // 5 bytes from start of call to where return address points
   564   if (_method_handle_invoke)
   565     offset += preserve_SP_size();
   566   return offset;
   567 }
   569 int MachCallDynamicJavaNode::ret_addr_offset()
   570 {
   571   return 15; // 15 bytes from start of call to where return address points
   572 }
   574 // In os_cpu .ad file
   575 // int MachCallRuntimeNode::ret_addr_offset()
   577 // Indicate if the safepoint node needs the polling page as an input.
   578 // Since amd64 does not have absolute addressing but RIP-relative
   579 // addressing and the polling page is within 2G, it doesn't.
   580 bool SafePointNode::needs_polling_address_input()
   581 {
   582   return false;
   583 }
   585 //
   586 // Compute padding required for nodes which need alignment
   587 //
   589 // The address of the call instruction needs to be 4-byte aligned to
   590 // ensure that it does not span a cache line so that it can be patched.
   591 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
   592 {
   593   current_offset += 1; // skip call opcode byte
   594   return round_to(current_offset, alignment_required()) - current_offset;
   595 }
   597 // The address of the call instruction needs to be 4-byte aligned to
   598 // ensure that it does not span a cache line so that it can be patched.
   599 int CallStaticJavaHandleNode::compute_padding(int current_offset) const
   600 {
   601   current_offset += preserve_SP_size();   // skip mov rbp, rsp
   602   current_offset += 1; // skip call opcode byte
   603   return round_to(current_offset, alignment_required()) - current_offset;
   604 }
   606 // The address of the call instruction needs to be 4-byte aligned to
   607 // ensure that it does not span a cache line so that it can be patched.
   608 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
   609 {
   610   current_offset += 11; // skip movq instruction + call opcode byte
   611   return round_to(current_offset, alignment_required()) - current_offset;
   612 }
   614 #ifndef PRODUCT
   615 void MachBreakpointNode::format(PhaseRegAlloc*, outputStream* st) const
   616 {
   617   st->print("INT3");
   618 }
   619 #endif
   621 // EMIT_RM()
   622 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
   623   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
   624   cbuf.insts()->emit_int8(c);
   625 }
   627 // EMIT_CC()
   628 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
   629   unsigned char c = (unsigned char) (f1 | f2);
   630   cbuf.insts()->emit_int8(c);
   631 }
   633 // EMIT_OPCODE()
   634 void emit_opcode(CodeBuffer &cbuf, int code) {
   635   cbuf.insts()->emit_int8((unsigned char) code);
   636 }
   638 // EMIT_OPCODE() w/ relocation information
   639 void emit_opcode(CodeBuffer &cbuf,
   640                  int code, relocInfo::relocType reloc, int offset, int format)
   641 {
   642   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
   643   emit_opcode(cbuf, code);
   644 }
   646 // EMIT_D8()
   647 void emit_d8(CodeBuffer &cbuf, int d8) {
   648   cbuf.insts()->emit_int8((unsigned char) d8);
   649 }
   651 // EMIT_D16()
   652 void emit_d16(CodeBuffer &cbuf, int d16) {
   653   cbuf.insts()->emit_int16(d16);
   654 }
   656 // EMIT_D32()
   657 void emit_d32(CodeBuffer &cbuf, int d32) {
   658   cbuf.insts()->emit_int32(d32);
   659 }
   661 // EMIT_D64()
   662 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
   663   cbuf.insts()->emit_int64(d64);
   664 }
   666 // emit 32 bit value and construct relocation entry from relocInfo::relocType
   667 void emit_d32_reloc(CodeBuffer& cbuf,
   668                     int d32,
   669                     relocInfo::relocType reloc,
   670                     int format)
   671 {
   672   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
   673   cbuf.relocate(cbuf.insts_mark(), reloc, format);
   674   cbuf.insts()->emit_int32(d32);
   675 }
   677 // emit 32 bit value and construct relocation entry from RelocationHolder
   678 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
   679 #ifdef ASSERT
   680   if (rspec.reloc()->type() == relocInfo::oop_type &&
   681       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
   682     assert(oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
   683   }
   684 #endif
   685   cbuf.relocate(cbuf.insts_mark(), rspec, format);
   686   cbuf.insts()->emit_int32(d32);
   687 }
   689 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
   690   address next_ip = cbuf.insts_end() + 4;
   691   emit_d32_reloc(cbuf, (int) (addr - next_ip),
   692                  external_word_Relocation::spec(addr),
   693                  RELOC_DISP32);
   694 }
   697 // emit 64 bit value and construct relocation entry from relocInfo::relocType
   698 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
   699   cbuf.relocate(cbuf.insts_mark(), reloc, format);
   700   cbuf.insts()->emit_int64(d64);
   701 }
   703 // emit 64 bit value and construct relocation entry from RelocationHolder
   704 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
   705 #ifdef ASSERT
   706   if (rspec.reloc()->type() == relocInfo::oop_type &&
   707       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
   708     assert(oop(d64)->is_oop() && (ScavengeRootsInCode || !oop(d64)->is_scavengable()),
   709            "cannot embed scavengable oops in code");
   710   }
   711 #endif
   712   cbuf.relocate(cbuf.insts_mark(), rspec, format);
   713   cbuf.insts()->emit_int64(d64);
   714 }
   716 // Access stack slot for load or store
   717 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
   718 {
   719   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
   720   if (-0x80 <= disp && disp < 0x80) {
   721     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
   722     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
   723     emit_d8(cbuf, disp);     // Displacement  // R/M byte
   724   } else {
   725     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
   726     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
   727     emit_d32(cbuf, disp);     // Displacement // R/M byte
   728   }
   729 }
   731    // rRegI ereg, memory mem) %{    // emit_reg_mem
   732 void encode_RegMem(CodeBuffer &cbuf,
   733                    int reg,
   734                    int base, int index, int scale, int disp, bool disp_is_oop)
   735 {
   736   assert(!disp_is_oop, "cannot have disp");
   737   int regenc = reg & 7;
   738   int baseenc = base & 7;
   739   int indexenc = index & 7;
   741   // There is no index & no scale, use form without SIB byte
   742   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
   743     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
   744     if (disp == 0 && base != RBP_enc && base != R13_enc) {
   745       emit_rm(cbuf, 0x0, regenc, baseenc); // *
   746     } else if (-0x80 <= disp && disp < 0x80 && !disp_is_oop) {
   747       // If 8-bit displacement, mode 0x1
   748       emit_rm(cbuf, 0x1, regenc, baseenc); // *
   749       emit_d8(cbuf, disp);
   750     } else {
   751       // If 32-bit displacement
   752       if (base == -1) { // Special flag for absolute address
   753         emit_rm(cbuf, 0x0, regenc, 0x5); // *
   754         if (disp_is_oop) {
   755           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
   756         } else {
   757           emit_d32(cbuf, disp);
   758         }
   759       } else {
   760         // Normal base + offset
   761         emit_rm(cbuf, 0x2, regenc, baseenc); // *
   762         if (disp_is_oop) {
   763           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
   764         } else {
   765           emit_d32(cbuf, disp);
   766         }
   767       }
   768     }
   769   } else {
   770     // Else, encode with the SIB byte
   771     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
   772     if (disp == 0 && base != RBP_enc && base != R13_enc) {
   773       // If no displacement
   774       emit_rm(cbuf, 0x0, regenc, 0x4); // *
   775       emit_rm(cbuf, scale, indexenc, baseenc);
   776     } else {
   777       if (-0x80 <= disp && disp < 0x80 && !disp_is_oop) {
   778         // If 8-bit displacement, mode 0x1
   779         emit_rm(cbuf, 0x1, regenc, 0x4); // *
   780         emit_rm(cbuf, scale, indexenc, baseenc);
   781         emit_d8(cbuf, disp);
   782       } else {
   783         // If 32-bit displacement
   784         if (base == 0x04 ) {
   785           emit_rm(cbuf, 0x2, regenc, 0x4);
   786           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
   787         } else {
   788           emit_rm(cbuf, 0x2, regenc, 0x4);
   789           emit_rm(cbuf, scale, indexenc, baseenc); // *
   790         }
   791         if (disp_is_oop) {
   792           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
   793         } else {
   794           emit_d32(cbuf, disp);
   795         }
   796       }
   797     }
   798   }
   799 }
   801 void encode_copy(CodeBuffer &cbuf, int dstenc, int srcenc)
   802 {
   803   if (dstenc != srcenc) {
   804     if (dstenc < 8) {
   805       if (srcenc >= 8) {
   806         emit_opcode(cbuf, Assembler::REX_B);
   807         srcenc -= 8;
   808       }
   809     } else {
   810       if (srcenc < 8) {
   811         emit_opcode(cbuf, Assembler::REX_R);
   812       } else {
   813         emit_opcode(cbuf, Assembler::REX_RB);
   814         srcenc -= 8;
   815       }
   816       dstenc -= 8;
   817     }
   819     emit_opcode(cbuf, 0x8B);
   820     emit_rm(cbuf, 0x3, dstenc, srcenc);
   821   }
   822 }
   824 void encode_CopyXD( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {
   825   if( dst_encoding == src_encoding ) {
   826     // reg-reg copy, use an empty encoding
   827   } else {
   828     MacroAssembler _masm(&cbuf);
   830     __ movdqa(as_XMMRegister(dst_encoding), as_XMMRegister(src_encoding));
   831   }
   832 }
   835 //=============================================================================
   836 #ifndef PRODUCT
   837 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const
   838 {
   839   Compile* C = ra_->C;
   841   int framesize = C->frame_slots() << LogBytesPerInt;
   842   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
   843   // Remove wordSize for return adr already pushed
   844   // and another for the RBP we are going to save
   845   framesize -= 2*wordSize;
   846   bool need_nop = true;
   848   // Calls to C2R adapters often do not accept exceptional returns.
   849   // We require that their callers must bang for them.  But be
   850   // careful, because some VM calls (such as call site linkage) can
   851   // use several kilobytes of stack.  But the stack safety zone should
   852   // account for that.  See bugs 4446381, 4468289, 4497237.
   853   if (C->need_stack_bang(framesize)) {
   854     st->print_cr("# stack bang"); st->print("\t");
   855     need_nop = false;
   856   }
   857   st->print_cr("pushq   rbp"); st->print("\t");
   859   if (VerifyStackAtCalls) {
   860     // Majik cookie to verify stack depth
   861     st->print_cr("pushq   0xffffffffbadb100d"
   862                   "\t# Majik cookie for stack depth check");
   863     st->print("\t");
   864     framesize -= wordSize; // Remove 2 for cookie
   865     need_nop = false;
   866   }
   868   if (framesize) {
   869     st->print("subq    rsp, #%d\t# Create frame", framesize);
   870     if (framesize < 0x80 && need_nop) {
   871       st->print("\n\tnop\t# nop for patch_verified_entry");
   872     }
   873   }
   874 }
   875 #endif
   877 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const
   878 {
   879   Compile* C = ra_->C;
   881   // WARNING: Initial instruction MUST be 5 bytes or longer so that
   882   // NativeJump::patch_verified_entry will be able to patch out the entry
   883   // code safely. The fldcw is ok at 6 bytes, the push to verify stack
   884   // depth is ok at 5 bytes, the frame allocation can be either 3 or
   885   // 6 bytes. So if we don't do the fldcw or the push then we must
   886   // use the 6 byte frame allocation even if we have no frame. :-(
   887   // If method sets FPU control word do it now
   889   int framesize = C->frame_slots() << LogBytesPerInt;
   890   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
   891   // Remove wordSize for return adr already pushed
   892   // and another for the RBP we are going to save
   893   framesize -= 2*wordSize;
   894   bool need_nop = true;
   896   // Calls to C2R adapters often do not accept exceptional returns.
   897   // We require that their callers must bang for them.  But be
   898   // careful, because some VM calls (such as call site linkage) can
   899   // use several kilobytes of stack.  But the stack safety zone should
   900   // account for that.  See bugs 4446381, 4468289, 4497237.
   901   if (C->need_stack_bang(framesize)) {
   902     MacroAssembler masm(&cbuf);
   903     masm.generate_stack_overflow_check(framesize);
   904     need_nop = false;
   905   }
   907   // We always push rbp so that on return to interpreter rbp will be
   908   // restored correctly and we can correct the stack.
   909   emit_opcode(cbuf, 0x50 | RBP_enc);
   911   if (VerifyStackAtCalls) {
   912     // Majik cookie to verify stack depth
   913     emit_opcode(cbuf, 0x68); // pushq (sign-extended) 0xbadb100d
   914     emit_d32(cbuf, 0xbadb100d);
   915     framesize -= wordSize; // Remove 2 for cookie
   916     need_nop = false;
   917   }
   919   if (framesize) {
   920     emit_opcode(cbuf, Assembler::REX_W);
   921     if (framesize < 0x80) {
   922       emit_opcode(cbuf, 0x83);   // sub  SP,#framesize
   923       emit_rm(cbuf, 0x3, 0x05, RSP_enc);
   924       emit_d8(cbuf, framesize);
   925       if (need_nop) {
   926         emit_opcode(cbuf, 0x90); // nop
   927       }
   928     } else {
   929       emit_opcode(cbuf, 0x81);   // sub  SP,#framesize
   930       emit_rm(cbuf, 0x3, 0x05, RSP_enc);
   931       emit_d32(cbuf, framesize);
   932     }
   933   }
   935   C->set_frame_complete(cbuf.insts_size());
   937 #ifdef ASSERT
   938   if (VerifyStackAtCalls) {
   939     Label L;
   940     MacroAssembler masm(&cbuf);
   941     masm.push(rax);
   942     masm.mov(rax, rsp);
   943     masm.andptr(rax, StackAlignmentInBytes-1);
   944     masm.cmpptr(rax, StackAlignmentInBytes-wordSize);
   945     masm.pop(rax);
   946     masm.jcc(Assembler::equal, L);
   947     masm.stop("Stack is not properly aligned!");
   948     masm.bind(L);
   949   }
   950 #endif
   951 }
   953 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
   954 {
   955   return MachNode::size(ra_); // too many variables; just compute it
   956                               // the hard way
   957 }
   959 int MachPrologNode::reloc() const
   960 {
   961   return 0; // a large enough number
   962 }
   964 //=============================================================================
   965 #ifndef PRODUCT
   966 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
   967 {
   968   Compile* C = ra_->C;
   969   int framesize = C->frame_slots() << LogBytesPerInt;
   970   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
   971   // Remove word for return adr already pushed
   972   // and RBP
   973   framesize -= 2*wordSize;
   975   if (framesize) {
   976     st->print_cr("addq\trsp, %d\t# Destroy frame", framesize);
   977     st->print("\t");
   978   }
   980   st->print_cr("popq\trbp");
   981   if (do_polling() && C->is_method_compilation()) {
   982     st->print_cr("\ttestl\trax, [rip + #offset_to_poll_page]\t"
   983                   "# Safepoint: poll for GC");
   984     st->print("\t");
   985   }
   986 }
   987 #endif
   989 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
   990 {
   991   Compile* C = ra_->C;
   992   int framesize = C->frame_slots() << LogBytesPerInt;
   993   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
   994   // Remove word for return adr already pushed
   995   // and RBP
   996   framesize -= 2*wordSize;
   998   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  1000   if (framesize) {
  1001     emit_opcode(cbuf, Assembler::REX_W);
  1002     if (framesize < 0x80) {
  1003       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
  1004       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  1005       emit_d8(cbuf, framesize);
  1006     } else {
  1007       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
  1008       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  1009       emit_d32(cbuf, framesize);
  1013   // popq rbp
  1014   emit_opcode(cbuf, 0x58 | RBP_enc);
  1016   if (do_polling() && C->is_method_compilation()) {
  1017     // testl %rax, off(%rip) // Opcode + ModRM + Disp32 == 6 bytes
  1018     // XXX reg_mem doesn't support RIP-relative addressing yet
  1019     cbuf.set_insts_mark();
  1020     cbuf.relocate(cbuf.insts_mark(), relocInfo::poll_return_type, 0); // XXX
  1021     emit_opcode(cbuf, 0x85); // testl
  1022     emit_rm(cbuf, 0x0, RAX_enc, 0x5); // 00 rax 101 == 0x5
  1023     // cbuf.insts_mark() is beginning of instruction
  1024     emit_d32_reloc(cbuf, os::get_polling_page());
  1025 //                    relocInfo::poll_return_type,
  1029 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  1031   Compile* C = ra_->C;
  1032   int framesize = C->frame_slots() << LogBytesPerInt;
  1033   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  1034   // Remove word for return adr already pushed
  1035   // and RBP
  1036   framesize -= 2*wordSize;
  1038   uint size = 0;
  1040   if (do_polling() && C->is_method_compilation()) {
  1041     size += 6;
  1044   // count popq rbp
  1045   size++;
  1047   if (framesize) {
  1048     if (framesize < 0x80) {
  1049       size += 4;
  1050     } else if (framesize) {
  1051       size += 7;
  1055   return size;
  1058 int MachEpilogNode::reloc() const
  1060   return 2; // a large enough number
  1063 const Pipeline* MachEpilogNode::pipeline() const
  1065   return MachNode::pipeline_class();
  1068 int MachEpilogNode::safepoint_offset() const
  1070   return 0;
  1073 //=============================================================================
  1075 enum RC {
  1076   rc_bad,
  1077   rc_int,
  1078   rc_float,
  1079   rc_stack
  1080 };
  1082 static enum RC rc_class(OptoReg::Name reg)
  1084   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  1086   if (OptoReg::is_stack(reg)) return rc_stack;
  1088   VMReg r = OptoReg::as_VMReg(reg);
  1090   if (r->is_Register()) return rc_int;
  1092   assert(r->is_XMMRegister(), "must be");
  1093   return rc_float;
  1096 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
  1097                                        PhaseRegAlloc* ra_,
  1098                                        bool do_size,
  1099                                        outputStream* st) const
  1102   // Get registers to move
  1103   OptoReg::Name src_second = ra_->get_reg_second(in(1));
  1104   OptoReg::Name src_first = ra_->get_reg_first(in(1));
  1105   OptoReg::Name dst_second = ra_->get_reg_second(this);
  1106   OptoReg::Name dst_first = ra_->get_reg_first(this);
  1108   enum RC src_second_rc = rc_class(src_second);
  1109   enum RC src_first_rc = rc_class(src_first);
  1110   enum RC dst_second_rc = rc_class(dst_second);
  1111   enum RC dst_first_rc = rc_class(dst_first);
  1113   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
  1114          "must move at least 1 register" );
  1116   if (src_first == dst_first && src_second == dst_second) {
  1117     // Self copy, no move
  1118     return 0;
  1119   } else if (src_first_rc == rc_stack) {
  1120     // mem ->
  1121     if (dst_first_rc == rc_stack) {
  1122       // mem -> mem
  1123       assert(src_second != dst_first, "overlap");
  1124       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
  1125           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
  1126         // 64-bit
  1127         int src_offset = ra_->reg2offset(src_first);
  1128         int dst_offset = ra_->reg2offset(dst_first);
  1129         if (cbuf) {
  1130           emit_opcode(*cbuf, 0xFF);
  1131           encode_RegMem(*cbuf, RSI_enc, RSP_enc, 0x4, 0, src_offset, false);
  1133           emit_opcode(*cbuf, 0x8F);
  1134           encode_RegMem(*cbuf, RAX_enc, RSP_enc, 0x4, 0, dst_offset, false);
  1136 #ifndef PRODUCT
  1137         } else if (!do_size) {
  1138           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
  1139                      "popq    [rsp + #%d]",
  1140                      src_offset,
  1141                      dst_offset);
  1142 #endif
  1144         return
  1145           3 + ((src_offset == 0) ? 0 : (src_offset < 0x80 ? 1 : 4)) +
  1146           3 + ((dst_offset == 0) ? 0 : (dst_offset < 0x80 ? 1 : 4));
  1147       } else {
  1148         // 32-bit
  1149         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
  1150         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
  1151         // No pushl/popl, so:
  1152         int src_offset = ra_->reg2offset(src_first);
  1153         int dst_offset = ra_->reg2offset(dst_first);
  1154         if (cbuf) {
  1155           emit_opcode(*cbuf, Assembler::REX_W);
  1156           emit_opcode(*cbuf, 0x89);
  1157           emit_opcode(*cbuf, 0x44);
  1158           emit_opcode(*cbuf, 0x24);
  1159           emit_opcode(*cbuf, 0xF8);
  1161           emit_opcode(*cbuf, 0x8B);
  1162           encode_RegMem(*cbuf,
  1163                         RAX_enc,
  1164                         RSP_enc, 0x4, 0, src_offset,
  1165                         false);
  1167           emit_opcode(*cbuf, 0x89);
  1168           encode_RegMem(*cbuf,
  1169                         RAX_enc,
  1170                         RSP_enc, 0x4, 0, dst_offset,
  1171                         false);
  1173           emit_opcode(*cbuf, Assembler::REX_W);
  1174           emit_opcode(*cbuf, 0x8B);
  1175           emit_opcode(*cbuf, 0x44);
  1176           emit_opcode(*cbuf, 0x24);
  1177           emit_opcode(*cbuf, 0xF8);
  1179 #ifndef PRODUCT
  1180         } else if (!do_size) {
  1181           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
  1182                      "movl    rax, [rsp + #%d]\n\t"
  1183                      "movl    [rsp + #%d], rax\n\t"
  1184                      "movq    rax, [rsp - #8]",
  1185                      src_offset,
  1186                      dst_offset);
  1187 #endif
  1189         return
  1190           5 + // movq
  1191           3 + ((src_offset == 0) ? 0 : (src_offset < 0x80 ? 1 : 4)) + // movl
  1192           3 + ((dst_offset == 0) ? 0 : (dst_offset < 0x80 ? 1 : 4)) + // movl
  1193           5; // movq
  1195     } else if (dst_first_rc == rc_int) {
  1196       // mem -> gpr
  1197       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
  1198           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
  1199         // 64-bit
  1200         int offset = ra_->reg2offset(src_first);
  1201         if (cbuf) {
  1202           if (Matcher::_regEncode[dst_first] < 8) {
  1203             emit_opcode(*cbuf, Assembler::REX_W);
  1204           } else {
  1205             emit_opcode(*cbuf, Assembler::REX_WR);
  1207           emit_opcode(*cbuf, 0x8B);
  1208           encode_RegMem(*cbuf,
  1209                         Matcher::_regEncode[dst_first],
  1210                         RSP_enc, 0x4, 0, offset,
  1211                         false);
  1212 #ifndef PRODUCT
  1213         } else if (!do_size) {
  1214           st->print("movq    %s, [rsp + #%d]\t# spill",
  1215                      Matcher::regName[dst_first],
  1216                      offset);
  1217 #endif
  1219         return
  1220           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) + 4; // REX
  1221       } else {
  1222         // 32-bit
  1223         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
  1224         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
  1225         int offset = ra_->reg2offset(src_first);
  1226         if (cbuf) {
  1227           if (Matcher::_regEncode[dst_first] >= 8) {
  1228             emit_opcode(*cbuf, Assembler::REX_R);
  1230           emit_opcode(*cbuf, 0x8B);
  1231           encode_RegMem(*cbuf,
  1232                         Matcher::_regEncode[dst_first],
  1233                         RSP_enc, 0x4, 0, offset,
  1234                         false);
  1235 #ifndef PRODUCT
  1236         } else if (!do_size) {
  1237           st->print("movl    %s, [rsp + #%d]\t# spill",
  1238                      Matcher::regName[dst_first],
  1239                      offset);
  1240 #endif
  1242         return
  1243           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
  1244           ((Matcher::_regEncode[dst_first] < 8)
  1245            ? 3
  1246            : 4); // REX
  1248     } else if (dst_first_rc == rc_float) {
  1249       // mem-> xmm
  1250       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
  1251           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
  1252         // 64-bit
  1253         int offset = ra_->reg2offset(src_first);
  1254         if (cbuf) {
  1255           emit_opcode(*cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
  1256           if (Matcher::_regEncode[dst_first] >= 8) {
  1257             emit_opcode(*cbuf, Assembler::REX_R);
  1259           emit_opcode(*cbuf, 0x0F);
  1260           emit_opcode(*cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);
  1261           encode_RegMem(*cbuf,
  1262                         Matcher::_regEncode[dst_first],
  1263                         RSP_enc, 0x4, 0, offset,
  1264                         false);
  1265 #ifndef PRODUCT
  1266         } else if (!do_size) {
  1267           st->print("%s  %s, [rsp + #%d]\t# spill",
  1268                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
  1269                      Matcher::regName[dst_first],
  1270                      offset);
  1271 #endif
  1273         return
  1274           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
  1275           ((Matcher::_regEncode[dst_first] < 8)
  1276            ? 5
  1277            : 6); // REX
  1278       } else {
  1279         // 32-bit
  1280         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
  1281         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
  1282         int offset = ra_->reg2offset(src_first);
  1283         if (cbuf) {
  1284           emit_opcode(*cbuf, 0xF3);
  1285           if (Matcher::_regEncode[dst_first] >= 8) {
  1286             emit_opcode(*cbuf, Assembler::REX_R);
  1288           emit_opcode(*cbuf, 0x0F);
  1289           emit_opcode(*cbuf, 0x10);
  1290           encode_RegMem(*cbuf,
  1291                         Matcher::_regEncode[dst_first],
  1292                         RSP_enc, 0x4, 0, offset,
  1293                         false);
  1294 #ifndef PRODUCT
  1295         } else if (!do_size) {
  1296           st->print("movss   %s, [rsp + #%d]\t# spill",
  1297                      Matcher::regName[dst_first],
  1298                      offset);
  1299 #endif
  1301         return
  1302           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
  1303           ((Matcher::_regEncode[dst_first] < 8)
  1304            ? 5
  1305            : 6); // REX
  1308   } else if (src_first_rc == rc_int) {
  1309     // gpr ->
  1310     if (dst_first_rc == rc_stack) {
  1311       // gpr -> mem
  1312       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
  1313           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
  1314         // 64-bit
  1315         int offset = ra_->reg2offset(dst_first);
  1316         if (cbuf) {
  1317           if (Matcher::_regEncode[src_first] < 8) {
  1318             emit_opcode(*cbuf, Assembler::REX_W);
  1319           } else {
  1320             emit_opcode(*cbuf, Assembler::REX_WR);
  1322           emit_opcode(*cbuf, 0x89);
  1323           encode_RegMem(*cbuf,
  1324                         Matcher::_regEncode[src_first],
  1325                         RSP_enc, 0x4, 0, offset,
  1326                         false);
  1327 #ifndef PRODUCT
  1328         } else if (!do_size) {
  1329           st->print("movq    [rsp + #%d], %s\t# spill",
  1330                      offset,
  1331                      Matcher::regName[src_first]);
  1332 #endif
  1334         return ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) + 4; // REX
  1335       } else {
  1336         // 32-bit
  1337         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
  1338         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
  1339         int offset = ra_->reg2offset(dst_first);
  1340         if (cbuf) {
  1341           if (Matcher::_regEncode[src_first] >= 8) {
  1342             emit_opcode(*cbuf, Assembler::REX_R);
  1344           emit_opcode(*cbuf, 0x89);
  1345           encode_RegMem(*cbuf,
  1346                         Matcher::_regEncode[src_first],
  1347                         RSP_enc, 0x4, 0, offset,
  1348                         false);
  1349 #ifndef PRODUCT
  1350         } else if (!do_size) {
  1351           st->print("movl    [rsp + #%d], %s\t# spill",
  1352                      offset,
  1353                      Matcher::regName[src_first]);
  1354 #endif
  1356         return
  1357           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
  1358           ((Matcher::_regEncode[src_first] < 8)
  1359            ? 3
  1360            : 4); // REX
  1362     } else if (dst_first_rc == rc_int) {
  1363       // gpr -> gpr
  1364       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
  1365           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
  1366         // 64-bit
  1367         if (cbuf) {
  1368           if (Matcher::_regEncode[dst_first] < 8) {
  1369             if (Matcher::_regEncode[src_first] < 8) {
  1370               emit_opcode(*cbuf, Assembler::REX_W);
  1371             } else {
  1372               emit_opcode(*cbuf, Assembler::REX_WB);
  1374           } else {
  1375             if (Matcher::_regEncode[src_first] < 8) {
  1376               emit_opcode(*cbuf, Assembler::REX_WR);
  1377             } else {
  1378               emit_opcode(*cbuf, Assembler::REX_WRB);
  1381           emit_opcode(*cbuf, 0x8B);
  1382           emit_rm(*cbuf, 0x3,
  1383                   Matcher::_regEncode[dst_first] & 7,
  1384                   Matcher::_regEncode[src_first] & 7);
  1385 #ifndef PRODUCT
  1386         } else if (!do_size) {
  1387           st->print("movq    %s, %s\t# spill",
  1388                      Matcher::regName[dst_first],
  1389                      Matcher::regName[src_first]);
  1390 #endif
  1392         return 3; // REX
  1393       } else {
  1394         // 32-bit
  1395         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
  1396         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
  1397         if (cbuf) {
  1398           if (Matcher::_regEncode[dst_first] < 8) {
  1399             if (Matcher::_regEncode[src_first] >= 8) {
  1400               emit_opcode(*cbuf, Assembler::REX_B);
  1402           } else {
  1403             if (Matcher::_regEncode[src_first] < 8) {
  1404               emit_opcode(*cbuf, Assembler::REX_R);
  1405             } else {
  1406               emit_opcode(*cbuf, Assembler::REX_RB);
  1409           emit_opcode(*cbuf, 0x8B);
  1410           emit_rm(*cbuf, 0x3,
  1411                   Matcher::_regEncode[dst_first] & 7,
  1412                   Matcher::_regEncode[src_first] & 7);
  1413 #ifndef PRODUCT
  1414         } else if (!do_size) {
  1415           st->print("movl    %s, %s\t# spill",
  1416                      Matcher::regName[dst_first],
  1417                      Matcher::regName[src_first]);
  1418 #endif
  1420         return
  1421           (Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
  1422           ? 2
  1423           : 3; // REX
  1425     } else if (dst_first_rc == rc_float) {
  1426       // gpr -> xmm
  1427       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
  1428           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
  1429         // 64-bit
  1430         if (cbuf) {
  1431           emit_opcode(*cbuf, 0x66);
  1432           if (Matcher::_regEncode[dst_first] < 8) {
  1433             if (Matcher::_regEncode[src_first] < 8) {
  1434               emit_opcode(*cbuf, Assembler::REX_W);
  1435             } else {
  1436               emit_opcode(*cbuf, Assembler::REX_WB);
  1438           } else {
  1439             if (Matcher::_regEncode[src_first] < 8) {
  1440               emit_opcode(*cbuf, Assembler::REX_WR);
  1441             } else {
  1442               emit_opcode(*cbuf, Assembler::REX_WRB);
  1445           emit_opcode(*cbuf, 0x0F);
  1446           emit_opcode(*cbuf, 0x6E);
  1447           emit_rm(*cbuf, 0x3,
  1448                   Matcher::_regEncode[dst_first] & 7,
  1449                   Matcher::_regEncode[src_first] & 7);
  1450 #ifndef PRODUCT
  1451         } else if (!do_size) {
  1452           st->print("movdq   %s, %s\t# spill",
  1453                      Matcher::regName[dst_first],
  1454                      Matcher::regName[src_first]);
  1455 #endif
  1457         return 5; // REX
  1458       } else {
  1459         // 32-bit
  1460         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
  1461         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
  1462         if (cbuf) {
  1463           emit_opcode(*cbuf, 0x66);
  1464           if (Matcher::_regEncode[dst_first] < 8) {
  1465             if (Matcher::_regEncode[src_first] >= 8) {
  1466               emit_opcode(*cbuf, Assembler::REX_B);
  1468           } else {
  1469             if (Matcher::_regEncode[src_first] < 8) {
  1470               emit_opcode(*cbuf, Assembler::REX_R);
  1471             } else {
  1472               emit_opcode(*cbuf, Assembler::REX_RB);
  1475           emit_opcode(*cbuf, 0x0F);
  1476           emit_opcode(*cbuf, 0x6E);
  1477           emit_rm(*cbuf, 0x3,
  1478                   Matcher::_regEncode[dst_first] & 7,
  1479                   Matcher::_regEncode[src_first] & 7);
  1480 #ifndef PRODUCT
  1481         } else if (!do_size) {
  1482           st->print("movdl   %s, %s\t# spill",
  1483                      Matcher::regName[dst_first],
  1484                      Matcher::regName[src_first]);
  1485 #endif
  1487         return
  1488           (Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
  1489           ? 4
  1490           : 5; // REX
  1493   } else if (src_first_rc == rc_float) {
  1494     // xmm ->
  1495     if (dst_first_rc == rc_stack) {
  1496       // xmm -> mem
  1497       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
  1498           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
  1499         // 64-bit
  1500         int offset = ra_->reg2offset(dst_first);
  1501         if (cbuf) {
  1502           emit_opcode(*cbuf, 0xF2);
  1503           if (Matcher::_regEncode[src_first] >= 8) {
  1504               emit_opcode(*cbuf, Assembler::REX_R);
  1506           emit_opcode(*cbuf, 0x0F);
  1507           emit_opcode(*cbuf, 0x11);
  1508           encode_RegMem(*cbuf,
  1509                         Matcher::_regEncode[src_first],
  1510                         RSP_enc, 0x4, 0, offset,
  1511                         false);
  1512 #ifndef PRODUCT
  1513         } else if (!do_size) {
  1514           st->print("movsd   [rsp + #%d], %s\t# spill",
  1515                      offset,
  1516                      Matcher::regName[src_first]);
  1517 #endif
  1519         return
  1520           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
  1521           ((Matcher::_regEncode[src_first] < 8)
  1522            ? 5
  1523            : 6); // REX
  1524       } else {
  1525         // 32-bit
  1526         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
  1527         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
  1528         int offset = ra_->reg2offset(dst_first);
  1529         if (cbuf) {
  1530           emit_opcode(*cbuf, 0xF3);
  1531           if (Matcher::_regEncode[src_first] >= 8) {
  1532               emit_opcode(*cbuf, Assembler::REX_R);
  1534           emit_opcode(*cbuf, 0x0F);
  1535           emit_opcode(*cbuf, 0x11);
  1536           encode_RegMem(*cbuf,
  1537                         Matcher::_regEncode[src_first],
  1538                         RSP_enc, 0x4, 0, offset,
  1539                         false);
  1540 #ifndef PRODUCT
  1541         } else if (!do_size) {
  1542           st->print("movss   [rsp + #%d], %s\t# spill",
  1543                      offset,
  1544                      Matcher::regName[src_first]);
  1545 #endif
  1547         return
  1548           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
  1549           ((Matcher::_regEncode[src_first] < 8)
  1550            ? 5
  1551            : 6); // REX
  1553     } else if (dst_first_rc == rc_int) {
  1554       // xmm -> gpr
  1555       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
  1556           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
  1557         // 64-bit
  1558         if (cbuf) {
  1559           emit_opcode(*cbuf, 0x66);
  1560           if (Matcher::_regEncode[dst_first] < 8) {
  1561             if (Matcher::_regEncode[src_first] < 8) {
  1562               emit_opcode(*cbuf, Assembler::REX_W);
  1563             } else {
  1564               emit_opcode(*cbuf, Assembler::REX_WR); // attention!
  1566           } else {
  1567             if (Matcher::_regEncode[src_first] < 8) {
  1568               emit_opcode(*cbuf, Assembler::REX_WB); // attention!
  1569             } else {
  1570               emit_opcode(*cbuf, Assembler::REX_WRB);
  1573           emit_opcode(*cbuf, 0x0F);
  1574           emit_opcode(*cbuf, 0x7E);
  1575           emit_rm(*cbuf, 0x3,
  1576                   Matcher::_regEncode[src_first] & 7,
  1577                   Matcher::_regEncode[dst_first] & 7);
  1578 #ifndef PRODUCT
  1579         } else if (!do_size) {
  1580           st->print("movdq   %s, %s\t# spill",
  1581                      Matcher::regName[dst_first],
  1582                      Matcher::regName[src_first]);
  1583 #endif
  1585         return 5; // REX
  1586       } else {
  1587         // 32-bit
  1588         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
  1589         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
  1590         if (cbuf) {
  1591           emit_opcode(*cbuf, 0x66);
  1592           if (Matcher::_regEncode[dst_first] < 8) {
  1593             if (Matcher::_regEncode[src_first] >= 8) {
  1594               emit_opcode(*cbuf, Assembler::REX_R); // attention!
  1596           } else {
  1597             if (Matcher::_regEncode[src_first] < 8) {
  1598               emit_opcode(*cbuf, Assembler::REX_B); // attention!
  1599             } else {
  1600               emit_opcode(*cbuf, Assembler::REX_RB);
  1603           emit_opcode(*cbuf, 0x0F);
  1604           emit_opcode(*cbuf, 0x7E);
  1605           emit_rm(*cbuf, 0x3,
  1606                   Matcher::_regEncode[src_first] & 7,
  1607                   Matcher::_regEncode[dst_first] & 7);
  1608 #ifndef PRODUCT
  1609         } else if (!do_size) {
  1610           st->print("movdl   %s, %s\t# spill",
  1611                      Matcher::regName[dst_first],
  1612                      Matcher::regName[src_first]);
  1613 #endif
  1615         return
  1616           (Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
  1617           ? 4
  1618           : 5; // REX
  1620     } else if (dst_first_rc == rc_float) {
  1621       // xmm -> xmm
  1622       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
  1623           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
  1624         // 64-bit
  1625         if (cbuf) {
  1626           emit_opcode(*cbuf, UseXmmRegToRegMoveAll ? 0x66 : 0xF2);
  1627           if (Matcher::_regEncode[dst_first] < 8) {
  1628             if (Matcher::_regEncode[src_first] >= 8) {
  1629               emit_opcode(*cbuf, Assembler::REX_B);
  1631           } else {
  1632             if (Matcher::_regEncode[src_first] < 8) {
  1633               emit_opcode(*cbuf, Assembler::REX_R);
  1634             } else {
  1635               emit_opcode(*cbuf, Assembler::REX_RB);
  1638           emit_opcode(*cbuf, 0x0F);
  1639           emit_opcode(*cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
  1640           emit_rm(*cbuf, 0x3,
  1641                   Matcher::_regEncode[dst_first] & 7,
  1642                   Matcher::_regEncode[src_first] & 7);
  1643 #ifndef PRODUCT
  1644         } else if (!do_size) {
  1645           st->print("%s  %s, %s\t# spill",
  1646                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
  1647                      Matcher::regName[dst_first],
  1648                      Matcher::regName[src_first]);
  1649 #endif
  1651         return
  1652           (Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
  1653           ? 4
  1654           : 5; // REX
  1655       } else {
  1656         // 32-bit
  1657         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
  1658         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
  1659         if (cbuf) {
  1660           if (!UseXmmRegToRegMoveAll)
  1661             emit_opcode(*cbuf, 0xF3);
  1662           if (Matcher::_regEncode[dst_first] < 8) {
  1663             if (Matcher::_regEncode[src_first] >= 8) {
  1664               emit_opcode(*cbuf, Assembler::REX_B);
  1666           } else {
  1667             if (Matcher::_regEncode[src_first] < 8) {
  1668               emit_opcode(*cbuf, Assembler::REX_R);
  1669             } else {
  1670               emit_opcode(*cbuf, Assembler::REX_RB);
  1673           emit_opcode(*cbuf, 0x0F);
  1674           emit_opcode(*cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
  1675           emit_rm(*cbuf, 0x3,
  1676                   Matcher::_regEncode[dst_first] & 7,
  1677                   Matcher::_regEncode[src_first] & 7);
  1678 #ifndef PRODUCT
  1679         } else if (!do_size) {
  1680           st->print("%s  %s, %s\t# spill",
  1681                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
  1682                      Matcher::regName[dst_first],
  1683                      Matcher::regName[src_first]);
  1684 #endif
  1686         return
  1687           (Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
  1688           ? (UseXmmRegToRegMoveAll ? 3 : 4)
  1689           : (UseXmmRegToRegMoveAll ? 4 : 5); // REX
  1694   assert(0," foo ");
  1695   Unimplemented();
  1697   return 0;
  1700 #ifndef PRODUCT
  1701 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const
  1703   implementation(NULL, ra_, false, st);
  1705 #endif
  1707 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const
  1709   implementation(&cbuf, ra_, false, NULL);
  1712 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const
  1714   return implementation(NULL, ra_, true, NULL);
  1717 //=============================================================================
  1718 #ifndef PRODUCT
  1719 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const
  1721   st->print("nop \t# %d bytes pad for loops and calls", _count);
  1723 #endif
  1725 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const
  1727   MacroAssembler _masm(&cbuf);
  1728   __ nop(_count);
  1731 uint MachNopNode::size(PhaseRegAlloc*) const
  1733   return _count;
  1737 //=============================================================================
  1738 #ifndef PRODUCT
  1739 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  1741   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
  1742   int reg = ra_->get_reg_first(this);
  1743   st->print("leaq    %s, [rsp + #%d]\t# box lock",
  1744             Matcher::regName[reg], offset);
  1746 #endif
  1748 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  1750   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
  1751   int reg = ra_->get_encode(this);
  1752   if (offset >= 0x80) {
  1753     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
  1754     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
  1755     emit_rm(cbuf, 0x2, reg & 7, 0x04);
  1756     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
  1757     emit_d32(cbuf, offset);
  1758   } else {
  1759     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
  1760     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
  1761     emit_rm(cbuf, 0x1, reg & 7, 0x04);
  1762     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
  1763     emit_d8(cbuf, offset);
  1767 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
  1769   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
  1770   return (offset < 0x80) ? 5 : 8; // REX
  1773 //=============================================================================
  1775 // emit call stub, compiled java to interpreter
  1776 void emit_java_to_interp(CodeBuffer& cbuf)
  1778   // Stub is fixed up when the corresponding call is converted from
  1779   // calling compiled code to calling interpreted code.
  1780   // movq rbx, 0
  1781   // jmp -5 # to self
  1783   address mark = cbuf.insts_mark();  // get mark within main instrs section
  1785   // Note that the code buffer's insts_mark is always relative to insts.
  1786   // That's why we must use the macroassembler to generate a stub.
  1787   MacroAssembler _masm(&cbuf);
  1789   address base =
  1790   __ start_a_stub(Compile::MAX_stubs_size);
  1791   if (base == NULL)  return;  // CodeBuffer::expand failed
  1792   // static stub relocation stores the instruction address of the call
  1793   __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM64);
  1794   // static stub relocation also tags the methodOop in the code-stream.
  1795   __ movoop(rbx, (jobject) NULL);  // method is zapped till fixup time
  1796   // This is recognized as unresolved by relocs/nativeinst/ic code
  1797   __ jump(RuntimeAddress(__ pc()));
  1799   // Update current stubs pointer and restore insts_end.
  1800   __ end_a_stub();
  1803 // size of call stub, compiled java to interpretor
  1804 uint size_java_to_interp()
  1806   return 15;  // movq (1+1+8); jmp (1+4)
  1809 // relocation entries for call stub, compiled java to interpretor
  1810 uint reloc_java_to_interp()
  1812   return 4; // 3 in emit_java_to_interp + 1 in Java_Static_Call
  1815 //=============================================================================
  1816 #ifndef PRODUCT
  1817 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  1819   if (UseCompressedOops) {
  1820     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
  1821     if (Universe::narrow_oop_shift() != 0) {
  1822       st->print_cr("\tdecode_heap_oop_not_null rscratch1, rscratch1");
  1824     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
  1825   } else {
  1826     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
  1827                  "# Inline cache check");
  1829   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
  1830   st->print_cr("\tnop\t# nops to align entry point");
  1832 #endif
  1834 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  1836   MacroAssembler masm(&cbuf);
  1837   uint insts_size = cbuf.insts_size();
  1838   if (UseCompressedOops) {
  1839     masm.load_klass(rscratch1, j_rarg0);
  1840     masm.cmpptr(rax, rscratch1);
  1841   } else {
  1842     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
  1845   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
  1847   /* WARNING these NOPs are critical so that verified entry point is properly
  1848      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
  1849   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
  1850   if (OptoBreakpoint) {
  1851     // Leave space for int3
  1852     nops_cnt -= 1;
  1854   nops_cnt &= 0x3; // Do not add nops if code is aligned.
  1855   if (nops_cnt > 0)
  1856     masm.nop(nops_cnt);
  1859 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
  1861   return MachNode::size(ra_); // too many variables; just compute it
  1862                               // the hard way
  1866 //=============================================================================
  1867 uint size_exception_handler()
  1869   // NativeCall instruction size is the same as NativeJump.
  1870   // Note that this value is also credited (in output.cpp) to
  1871   // the size of the code section.
  1872   return NativeJump::instruction_size;
  1875 // Emit exception handler code.
  1876 int emit_exception_handler(CodeBuffer& cbuf)
  1879   // Note that the code buffer's insts_mark is always relative to insts.
  1880   // That's why we must use the macroassembler to generate a handler.
  1881   MacroAssembler _masm(&cbuf);
  1882   address base =
  1883   __ start_a_stub(size_exception_handler());
  1884   if (base == NULL)  return 0;  // CodeBuffer::expand failed
  1885   int offset = __ offset();
  1886   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
  1887   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
  1888   __ end_a_stub();
  1889   return offset;
  1892 uint size_deopt_handler()
  1894   // three 5 byte instructions
  1895   return 15;
  1898 // Emit deopt handler code.
  1899 int emit_deopt_handler(CodeBuffer& cbuf)
  1902   // Note that the code buffer's insts_mark is always relative to insts.
  1903   // That's why we must use the macroassembler to generate a handler.
  1904   MacroAssembler _masm(&cbuf);
  1905   address base =
  1906   __ start_a_stub(size_deopt_handler());
  1907   if (base == NULL)  return 0;  // CodeBuffer::expand failed
  1908   int offset = __ offset();
  1909   address the_pc = (address) __ pc();
  1910   Label next;
  1911   // push a "the_pc" on the stack without destroying any registers
  1912   // as they all may be live.
  1914   // push address of "next"
  1915   __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
  1916   __ bind(next);
  1917   // adjust it so it matches "the_pc"
  1918   __ subptr(Address(rsp, 0), __ offset() - offset);
  1919   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
  1920   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
  1921   __ end_a_stub();
  1922   return offset;
  1925 static void emit_double_constant(CodeBuffer& cbuf, double x) {
  1926   int mark = cbuf.insts()->mark_off();
  1927   MacroAssembler _masm(&cbuf);
  1928   address double_address = __ double_constant(x);
  1929   cbuf.insts()->set_mark_off(mark);  // preserve mark across masm shift
  1930   emit_d32_reloc(cbuf,
  1931                  (int) (double_address - cbuf.insts_end() - 4),
  1932                  internal_word_Relocation::spec(double_address),
  1933                  RELOC_DISP32);
  1936 static void emit_float_constant(CodeBuffer& cbuf, float x) {
  1937   int mark = cbuf.insts()->mark_off();
  1938   MacroAssembler _masm(&cbuf);
  1939   address float_address = __ float_constant(x);
  1940   cbuf.insts()->set_mark_off(mark);  // preserve mark across masm shift
  1941   emit_d32_reloc(cbuf,
  1942                  (int) (float_address - cbuf.insts_end() - 4),
  1943                  internal_word_Relocation::spec(float_address),
  1944                  RELOC_DISP32);
  1948 const bool Matcher::match_rule_supported(int opcode) {
  1949   if (!has_match_rule(opcode))
  1950     return false;
  1952   return true;  // Per default match rules are supported.
  1955 int Matcher::regnum_to_fpu_offset(int regnum)
  1957   return regnum - 32; // The FP registers are in the second chunk
  1960 // This is UltraSparc specific, true just means we have fast l2f conversion
  1961 const bool Matcher::convL2FSupported(void) {
  1962   return true;
  1965 // Vector width in bytes
  1966 const uint Matcher::vector_width_in_bytes(void) {
  1967   return 8;
  1970 // Vector ideal reg
  1971 const uint Matcher::vector_ideal_reg(void) {
  1972   return Op_RegD;
  1975 // Is this branch offset short enough that a short branch can be used?
  1976 //
  1977 // NOTE: If the platform does not provide any short branch variants, then
  1978 //       this method should return false for offset 0.
  1979 bool Matcher::is_short_branch_offset(int rule, int offset) {
  1980   // the short version of jmpConUCF2 contains multiple branches,
  1981   // making the reach slightly less
  1982   if (rule == jmpConUCF2_rule)
  1983     return (-126 <= offset && offset <= 125);
  1984   return (-128 <= offset && offset <= 127);
  1987 const bool Matcher::isSimpleConstant64(jlong value) {
  1988   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
  1989   //return value == (int) value;  // Cf. storeImmL and immL32.
  1991   // Probably always true, even if a temp register is required.
  1992   return true;
  1995 // The ecx parameter to rep stosq for the ClearArray node is in words.
  1996 const bool Matcher::init_array_count_is_in_bytes = false;
  1998 // Threshold size for cleararray.
  1999 const int Matcher::init_array_short_size = 8 * BytesPerLong;
  2001 // Should the Matcher clone shifts on addressing modes, expecting them
  2002 // to be subsumed into complex addressing expressions or compute them
  2003 // into registers?  True for Intel but false for most RISCs
  2004 const bool Matcher::clone_shift_expressions = true;
  2006 bool Matcher::narrow_oop_use_complex_address() {
  2007   assert(UseCompressedOops, "only for compressed oops code");
  2008   return (LogMinObjAlignmentInBytes <= 3);
  2011 // Is it better to copy float constants, or load them directly from
  2012 // memory?  Intel can load a float constant from a direct address,
  2013 // requiring no extra registers.  Most RISCs will have to materialize
  2014 // an address into a register first, so they would do better to copy
  2015 // the constant from stack.
  2016 const bool Matcher::rematerialize_float_constants = true; // XXX
  2018 // If CPU can load and store mis-aligned doubles directly then no
  2019 // fixup is needed.  Else we split the double into 2 integer pieces
  2020 // and move it piece-by-piece.  Only happens when passing doubles into
  2021 // C code as the Java calling convention forces doubles to be aligned.
  2022 const bool Matcher::misaligned_doubles_ok = true;
  2024 // No-op on amd64
  2025 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
  2027 // Advertise here if the CPU requires explicit rounding operations to
  2028 // implement the UseStrictFP mode.
  2029 const bool Matcher::strict_fp_requires_explicit_rounding = true;
  2031 // Are floats conerted to double when stored to stack during deoptimization?
  2032 // On x64 it is stored without convertion so we can use normal access.
  2033 bool Matcher::float_in_double() { return false; }
  2035 // Do ints take an entire long register or just half?
  2036 const bool Matcher::int_in_long = true;
  2038 // Return whether or not this register is ever used as an argument.
  2039 // This function is used on startup to build the trampoline stubs in
  2040 // generateOptoStub.  Registers not mentioned will be killed by the VM
  2041 // call in the trampoline, and arguments in those registers not be
  2042 // available to the callee.
  2043 bool Matcher::can_be_java_arg(int reg)
  2045   return
  2046     reg ==  RDI_num || reg ==  RDI_H_num ||
  2047     reg ==  RSI_num || reg ==  RSI_H_num ||
  2048     reg ==  RDX_num || reg ==  RDX_H_num ||
  2049     reg ==  RCX_num || reg ==  RCX_H_num ||
  2050     reg ==   R8_num || reg ==   R8_H_num ||
  2051     reg ==   R9_num || reg ==   R9_H_num ||
  2052     reg ==  R12_num || reg ==  R12_H_num ||
  2053     reg == XMM0_num || reg == XMM0_H_num ||
  2054     reg == XMM1_num || reg == XMM1_H_num ||
  2055     reg == XMM2_num || reg == XMM2_H_num ||
  2056     reg == XMM3_num || reg == XMM3_H_num ||
  2057     reg == XMM4_num || reg == XMM4_H_num ||
  2058     reg == XMM5_num || reg == XMM5_H_num ||
  2059     reg == XMM6_num || reg == XMM6_H_num ||
  2060     reg == XMM7_num || reg == XMM7_H_num;
  2063 bool Matcher::is_spillable_arg(int reg)
  2065   return can_be_java_arg(reg);
  2068 // Register for DIVI projection of divmodI
  2069 RegMask Matcher::divI_proj_mask() {
  2070   return INT_RAX_REG_mask;
  2073 // Register for MODI projection of divmodI
  2074 RegMask Matcher::modI_proj_mask() {
  2075   return INT_RDX_REG_mask;
  2078 // Register for DIVL projection of divmodL
  2079 RegMask Matcher::divL_proj_mask() {
  2080   return LONG_RAX_REG_mask;
  2083 // Register for MODL projection of divmodL
  2084 RegMask Matcher::modL_proj_mask() {
  2085   return LONG_RDX_REG_mask;
  2088 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
  2089   return PTR_RBP_REG_mask;
  2092 static Address build_address(int b, int i, int s, int d) {
  2093   Register index = as_Register(i);
  2094   Address::ScaleFactor scale = (Address::ScaleFactor)s;
  2095   if (index == rsp) {
  2096     index = noreg;
  2097     scale = Address::no_scale;
  2099   Address addr(as_Register(b), index, scale, d);
  2100   return addr;
  2103 %}
  2105 //----------ENCODING BLOCK-----------------------------------------------------
  2106 // This block specifies the encoding classes used by the compiler to
  2107 // output byte streams.  Encoding classes are parameterized macros
  2108 // used by Machine Instruction Nodes in order to generate the bit
  2109 // encoding of the instruction.  Operands specify their base encoding
  2110 // interface with the interface keyword.  There are currently
  2111 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
  2112 // COND_INTER.  REG_INTER causes an operand to generate a function
  2113 // which returns its register number when queried.  CONST_INTER causes
  2114 // an operand to generate a function which returns the value of the
  2115 // constant when queried.  MEMORY_INTER causes an operand to generate
  2116 // four functions which return the Base Register, the Index Register,
  2117 // the Scale Value, and the Offset Value of the operand when queried.
  2118 // COND_INTER causes an operand to generate six functions which return
  2119 // the encoding code (ie - encoding bits for the instruction)
  2120 // associated with each basic boolean condition for a conditional
  2121 // instruction.
  2122 //
  2123 // Instructions specify two basic values for encoding.  Again, a
  2124 // function is available to check if the constant displacement is an
  2125 // oop. They use the ins_encode keyword to specify their encoding
  2126 // classes (which must be a sequence of enc_class names, and their
  2127 // parameters, specified in the encoding block), and they use the
  2128 // opcode keyword to specify, in order, their primary, secondary, and
  2129 // tertiary opcode.  Only the opcode sections which a particular
  2130 // instruction needs for encoding need to be specified.
  2131 encode %{
  2132   // Build emit functions for each basic byte or larger field in the
  2133   // intel encoding scheme (opcode, rm, sib, immediate), and call them
  2134   // from C++ code in the enc_class source block.  Emit functions will
  2135   // live in the main source block for now.  In future, we can
  2136   // generalize this by adding a syntax that specifies the sizes of
  2137   // fields in an order, so that the adlc can build the emit functions
  2138   // automagically
  2140   // Emit primary opcode
  2141   enc_class OpcP
  2142   %{
  2143     emit_opcode(cbuf, $primary);
  2144   %}
  2146   // Emit secondary opcode
  2147   enc_class OpcS
  2148   %{
  2149     emit_opcode(cbuf, $secondary);
  2150   %}
  2152   // Emit tertiary opcode
  2153   enc_class OpcT
  2154   %{
  2155     emit_opcode(cbuf, $tertiary);
  2156   %}
  2158   // Emit opcode directly
  2159   enc_class Opcode(immI d8)
  2160   %{
  2161     emit_opcode(cbuf, $d8$$constant);
  2162   %}
  2164   // Emit size prefix
  2165   enc_class SizePrefix
  2166   %{
  2167     emit_opcode(cbuf, 0x66);
  2168   %}
  2170   enc_class reg(rRegI reg)
  2171   %{
  2172     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
  2173   %}
  2175   enc_class reg_reg(rRegI dst, rRegI src)
  2176   %{
  2177     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
  2178   %}
  2180   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
  2181   %{
  2182     emit_opcode(cbuf, $opcode$$constant);
  2183     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
  2184   %}
  2186   enc_class cmpfp_fixup()
  2187   %{
  2188     // jnp,s exit
  2189     emit_opcode(cbuf, 0x7B);
  2190     emit_d8(cbuf, 0x0A);
  2192     // pushfq
  2193     emit_opcode(cbuf, 0x9C);
  2195     // andq $0xffffff2b, (%rsp)
  2196     emit_opcode(cbuf, Assembler::REX_W);
  2197     emit_opcode(cbuf, 0x81);
  2198     emit_opcode(cbuf, 0x24);
  2199     emit_opcode(cbuf, 0x24);
  2200     emit_d32(cbuf, 0xffffff2b);
  2202     // popfq
  2203     emit_opcode(cbuf, 0x9D);
  2205     // nop (target for branch to avoid branch to branch)
  2206     emit_opcode(cbuf, 0x90);
  2207   %}
  2209   enc_class cmpfp3(rRegI dst)
  2210   %{
  2211     int dstenc = $dst$$reg;
  2213     // movl $dst, -1
  2214     if (dstenc >= 8) {
  2215       emit_opcode(cbuf, Assembler::REX_B);
  2217     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
  2218     emit_d32(cbuf, -1);
  2220     // jp,s done
  2221     emit_opcode(cbuf, 0x7A);
  2222     emit_d8(cbuf, dstenc < 4 ? 0x08 : 0x0A);
  2224     // jb,s done
  2225     emit_opcode(cbuf, 0x72);
  2226     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
  2228     // setne $dst
  2229     if (dstenc >= 4) {
  2230       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
  2232     emit_opcode(cbuf, 0x0F);
  2233     emit_opcode(cbuf, 0x95);
  2234     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
  2236     // movzbl $dst, $dst
  2237     if (dstenc >= 4) {
  2238       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
  2240     emit_opcode(cbuf, 0x0F);
  2241     emit_opcode(cbuf, 0xB6);
  2242     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
  2243   %}
  2245   enc_class cdql_enc(no_rax_rdx_RegI div)
  2246   %{
  2247     // Full implementation of Java idiv and irem; checks for
  2248     // special case as described in JVM spec., p.243 & p.271.
  2249     //
  2250     //         normal case                           special case
  2251     //
  2252     // input : rax: dividend                         min_int
  2253     //         reg: divisor                          -1
  2254     //
  2255     // output: rax: quotient  (= rax idiv reg)       min_int
  2256     //         rdx: remainder (= rax irem reg)       0
  2257     //
  2258     //  Code sequnce:
  2259     //
  2260     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
  2261     //    5:   75 07/08                jne    e <normal>
  2262     //    7:   33 d2                   xor    %edx,%edx
  2263     //  [div >= 8 -> offset + 1]
  2264     //  [REX_B]
  2265     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
  2266     //    c:   74 03/04                je     11 <done>
  2267     // 000000000000000e <normal>:
  2268     //    e:   99                      cltd
  2269     //  [div >= 8 -> offset + 1]
  2270     //  [REX_B]
  2271     //    f:   f7 f9                   idiv   $div
  2272     // 0000000000000011 <done>:
  2274     // cmp    $0x80000000,%eax
  2275     emit_opcode(cbuf, 0x3d);
  2276     emit_d8(cbuf, 0x00);
  2277     emit_d8(cbuf, 0x00);
  2278     emit_d8(cbuf, 0x00);
  2279     emit_d8(cbuf, 0x80);
  2281     // jne    e <normal>
  2282     emit_opcode(cbuf, 0x75);
  2283     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
  2285     // xor    %edx,%edx
  2286     emit_opcode(cbuf, 0x33);
  2287     emit_d8(cbuf, 0xD2);
  2289     // cmp    $0xffffffffffffffff,%ecx
  2290     if ($div$$reg >= 8) {
  2291       emit_opcode(cbuf, Assembler::REX_B);
  2293     emit_opcode(cbuf, 0x83);
  2294     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
  2295     emit_d8(cbuf, 0xFF);
  2297     // je     11 <done>
  2298     emit_opcode(cbuf, 0x74);
  2299     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
  2301     // <normal>
  2302     // cltd
  2303     emit_opcode(cbuf, 0x99);
  2305     // idivl (note: must be emitted by the user of this rule)
  2306     // <done>
  2307   %}
  2309   enc_class cdqq_enc(no_rax_rdx_RegL div)
  2310   %{
  2311     // Full implementation of Java ldiv and lrem; checks for
  2312     // special case as described in JVM spec., p.243 & p.271.
  2313     //
  2314     //         normal case                           special case
  2315     //
  2316     // input : rax: dividend                         min_long
  2317     //         reg: divisor                          -1
  2318     //
  2319     // output: rax: quotient  (= rax idiv reg)       min_long
  2320     //         rdx: remainder (= rax irem reg)       0
  2321     //
  2322     //  Code sequnce:
  2323     //
  2324     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
  2325     //    7:   00 00 80
  2326     //    a:   48 39 d0                cmp    %rdx,%rax
  2327     //    d:   75 08                   jne    17 <normal>
  2328     //    f:   33 d2                   xor    %edx,%edx
  2329     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
  2330     //   15:   74 05                   je     1c <done>
  2331     // 0000000000000017 <normal>:
  2332     //   17:   48 99                   cqto
  2333     //   19:   48 f7 f9                idiv   $div
  2334     // 000000000000001c <done>:
  2336     // mov    $0x8000000000000000,%rdx
  2337     emit_opcode(cbuf, Assembler::REX_W);
  2338     emit_opcode(cbuf, 0xBA);
  2339     emit_d8(cbuf, 0x00);
  2340     emit_d8(cbuf, 0x00);
  2341     emit_d8(cbuf, 0x00);
  2342     emit_d8(cbuf, 0x00);
  2343     emit_d8(cbuf, 0x00);
  2344     emit_d8(cbuf, 0x00);
  2345     emit_d8(cbuf, 0x00);
  2346     emit_d8(cbuf, 0x80);
  2348     // cmp    %rdx,%rax
  2349     emit_opcode(cbuf, Assembler::REX_W);
  2350     emit_opcode(cbuf, 0x39);
  2351     emit_d8(cbuf, 0xD0);
  2353     // jne    17 <normal>
  2354     emit_opcode(cbuf, 0x75);
  2355     emit_d8(cbuf, 0x08);
  2357     // xor    %edx,%edx
  2358     emit_opcode(cbuf, 0x33);
  2359     emit_d8(cbuf, 0xD2);
  2361     // cmp    $0xffffffffffffffff,$div
  2362     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
  2363     emit_opcode(cbuf, 0x83);
  2364     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
  2365     emit_d8(cbuf, 0xFF);
  2367     // je     1e <done>
  2368     emit_opcode(cbuf, 0x74);
  2369     emit_d8(cbuf, 0x05);
  2371     // <normal>
  2372     // cqto
  2373     emit_opcode(cbuf, Assembler::REX_W);
  2374     emit_opcode(cbuf, 0x99);
  2376     // idivq (note: must be emitted by the user of this rule)
  2377     // <done>
  2378   %}
  2380   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
  2381   enc_class OpcSE(immI imm)
  2382   %{
  2383     // Emit primary opcode and set sign-extend bit
  2384     // Check for 8-bit immediate, and set sign extend bit in opcode
  2385     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
  2386       emit_opcode(cbuf, $primary | 0x02);
  2387     } else {
  2388       // 32-bit immediate
  2389       emit_opcode(cbuf, $primary);
  2391   %}
  2393   enc_class OpcSErm(rRegI dst, immI imm)
  2394   %{
  2395     // OpcSEr/m
  2396     int dstenc = $dst$$reg;
  2397     if (dstenc >= 8) {
  2398       emit_opcode(cbuf, Assembler::REX_B);
  2399       dstenc -= 8;
  2401     // Emit primary opcode and set sign-extend bit
  2402     // Check for 8-bit immediate, and set sign extend bit in opcode
  2403     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
  2404       emit_opcode(cbuf, $primary | 0x02);
  2405     } else {
  2406       // 32-bit immediate
  2407       emit_opcode(cbuf, $primary);
  2409     // Emit r/m byte with secondary opcode, after primary opcode.
  2410     emit_rm(cbuf, 0x3, $secondary, dstenc);
  2411   %}
  2413   enc_class OpcSErm_wide(rRegL dst, immI imm)
  2414   %{
  2415     // OpcSEr/m
  2416     int dstenc = $dst$$reg;
  2417     if (dstenc < 8) {
  2418       emit_opcode(cbuf, Assembler::REX_W);
  2419     } else {
  2420       emit_opcode(cbuf, Assembler::REX_WB);
  2421       dstenc -= 8;
  2423     // Emit primary opcode and set sign-extend bit
  2424     // Check for 8-bit immediate, and set sign extend bit in opcode
  2425     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
  2426       emit_opcode(cbuf, $primary | 0x02);
  2427     } else {
  2428       // 32-bit immediate
  2429       emit_opcode(cbuf, $primary);
  2431     // Emit r/m byte with secondary opcode, after primary opcode.
  2432     emit_rm(cbuf, 0x3, $secondary, dstenc);
  2433   %}
  2435   enc_class Con8or32(immI imm)
  2436   %{
  2437     // Check for 8-bit immediate, and set sign extend bit in opcode
  2438     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
  2439       $$$emit8$imm$$constant;
  2440     } else {
  2441       // 32-bit immediate
  2442       $$$emit32$imm$$constant;
  2444   %}
  2446   enc_class Lbl(label labl)
  2447   %{
  2448     // JMP, CALL
  2449     Label* l = $labl$$label;
  2450     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0);
  2451   %}
  2453   enc_class LblShort(label labl)
  2454   %{
  2455     // JMP, CALL
  2456     Label* l = $labl$$label;
  2457     int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
  2458     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
  2459     emit_d8(cbuf, disp);
  2460   %}
  2462   enc_class opc2_reg(rRegI dst)
  2463   %{
  2464     // BSWAP
  2465     emit_cc(cbuf, $secondary, $dst$$reg);
  2466   %}
  2468   enc_class opc3_reg(rRegI dst)
  2469   %{
  2470     // BSWAP
  2471     emit_cc(cbuf, $tertiary, $dst$$reg);
  2472   %}
  2474   enc_class reg_opc(rRegI div)
  2475   %{
  2476     // INC, DEC, IDIV, IMOD, JMP indirect, ...
  2477     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
  2478   %}
  2480   enc_class Jcc(cmpOp cop, label labl)
  2481   %{
  2482     // JCC
  2483     Label* l = $labl$$label;
  2484     $$$emit8$primary;
  2485     emit_cc(cbuf, $secondary, $cop$$cmpcode);
  2486     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0);
  2487   %}
  2489   enc_class JccShort (cmpOp cop, label labl)
  2490   %{
  2491   // JCC
  2492     Label *l = $labl$$label;
  2493     emit_cc(cbuf, $primary, $cop$$cmpcode);
  2494     int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
  2495     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
  2496     emit_d8(cbuf, disp);
  2497   %}
  2499   enc_class enc_cmov(cmpOp cop)
  2500   %{
  2501     // CMOV
  2502     $$$emit8$primary;
  2503     emit_cc(cbuf, $secondary, $cop$$cmpcode);
  2504   %}
  2506   enc_class enc_cmovf_branch(cmpOp cop, regF dst, regF src)
  2507   %{
  2508     // Invert sense of branch from sense of cmov
  2509     emit_cc(cbuf, 0x70, $cop$$cmpcode ^ 1);
  2510     emit_d8(cbuf, ($dst$$reg < 8 && $src$$reg < 8)
  2511                   ? (UseXmmRegToRegMoveAll ? 3 : 4)
  2512                   : (UseXmmRegToRegMoveAll ? 4 : 5) ); // REX
  2513     // UseXmmRegToRegMoveAll ? movaps(dst, src) : movss(dst, src)
  2514     if (!UseXmmRegToRegMoveAll) emit_opcode(cbuf, 0xF3);
  2515     if ($dst$$reg < 8) {
  2516       if ($src$$reg >= 8) {
  2517         emit_opcode(cbuf, Assembler::REX_B);
  2519     } else {
  2520       if ($src$$reg < 8) {
  2521         emit_opcode(cbuf, Assembler::REX_R);
  2522       } else {
  2523         emit_opcode(cbuf, Assembler::REX_RB);
  2526     emit_opcode(cbuf, 0x0F);
  2527     emit_opcode(cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
  2528     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
  2529   %}
  2531   enc_class enc_cmovd_branch(cmpOp cop, regD dst, regD src)
  2532   %{
  2533     // Invert sense of branch from sense of cmov
  2534     emit_cc(cbuf, 0x70, $cop$$cmpcode ^ 1);
  2535     emit_d8(cbuf, $dst$$reg < 8 && $src$$reg < 8 ? 4 : 5); // REX
  2537     //  UseXmmRegToRegMoveAll ? movapd(dst, src) : movsd(dst, src)
  2538     emit_opcode(cbuf, UseXmmRegToRegMoveAll ? 0x66 : 0xF2);
  2539     if ($dst$$reg < 8) {
  2540       if ($src$$reg >= 8) {
  2541         emit_opcode(cbuf, Assembler::REX_B);
  2543     } else {
  2544       if ($src$$reg < 8) {
  2545         emit_opcode(cbuf, Assembler::REX_R);
  2546       } else {
  2547         emit_opcode(cbuf, Assembler::REX_RB);
  2550     emit_opcode(cbuf, 0x0F);
  2551     emit_opcode(cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
  2552     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
  2553   %}
  2555   enc_class enc_PartialSubtypeCheck()
  2556   %{
  2557     Register Rrdi = as_Register(RDI_enc); // result register
  2558     Register Rrax = as_Register(RAX_enc); // super class
  2559     Register Rrcx = as_Register(RCX_enc); // killed
  2560     Register Rrsi = as_Register(RSI_enc); // sub class
  2561     Label miss;
  2562     const bool set_cond_codes = true;
  2564     MacroAssembler _masm(&cbuf);
  2565     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
  2566                                      NULL, &miss,
  2567                                      /*set_cond_codes:*/ true);
  2568     if ($primary) {
  2569       __ xorptr(Rrdi, Rrdi);
  2571     __ bind(miss);
  2572   %}
  2574   enc_class Java_To_Interpreter(method meth)
  2575   %{
  2576     // CALL Java_To_Interpreter
  2577     // This is the instruction starting address for relocation info.
  2578     cbuf.set_insts_mark();
  2579     $$$emit8$primary;
  2580     // CALL directly to the runtime
  2581     emit_d32_reloc(cbuf,
  2582                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
  2583                    runtime_call_Relocation::spec(),
  2584                    RELOC_DISP32);
  2585   %}
  2587   enc_class preserve_SP %{
  2588     debug_only(int off0 = cbuf.insts_size());
  2589     MacroAssembler _masm(&cbuf);
  2590     // RBP is preserved across all calls, even compiled calls.
  2591     // Use it to preserve RSP in places where the callee might change the SP.
  2592     __ movptr(rbp_mh_SP_save, rsp);
  2593     debug_only(int off1 = cbuf.insts_size());
  2594     assert(off1 - off0 == preserve_SP_size(), "correct size prediction");
  2595   %}
  2597   enc_class restore_SP %{
  2598     MacroAssembler _masm(&cbuf);
  2599     __ movptr(rsp, rbp_mh_SP_save);
  2600   %}
  2602   enc_class Java_Static_Call(method meth)
  2603   %{
  2604     // JAVA STATIC CALL
  2605     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
  2606     // determine who we intended to call.
  2607     cbuf.set_insts_mark();
  2608     $$$emit8$primary;
  2610     if (!_method) {
  2611       emit_d32_reloc(cbuf,
  2612                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
  2613                      runtime_call_Relocation::spec(),
  2614                      RELOC_DISP32);
  2615     } else if (_optimized_virtual) {
  2616       emit_d32_reloc(cbuf,
  2617                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
  2618                      opt_virtual_call_Relocation::spec(),
  2619                      RELOC_DISP32);
  2620     } else {
  2621       emit_d32_reloc(cbuf,
  2622                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
  2623                      static_call_Relocation::spec(),
  2624                      RELOC_DISP32);
  2626     if (_method) {
  2627       // Emit stub for static call
  2628       emit_java_to_interp(cbuf);
  2630   %}
  2632   enc_class Java_Dynamic_Call(method meth)
  2633   %{
  2634     // JAVA DYNAMIC CALL
  2635     // !!!!!
  2636     // Generate  "movq rax, -1", placeholder instruction to load oop-info
  2637     // emit_call_dynamic_prologue( cbuf );
  2638     cbuf.set_insts_mark();
  2640     // movq rax, -1
  2641     emit_opcode(cbuf, Assembler::REX_W);
  2642     emit_opcode(cbuf, 0xB8 | RAX_enc);
  2643     emit_d64_reloc(cbuf,
  2644                    (int64_t) Universe::non_oop_word(),
  2645                    oop_Relocation::spec_for_immediate(), RELOC_IMM64);
  2646     address virtual_call_oop_addr = cbuf.insts_mark();
  2647     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
  2648     // who we intended to call.
  2649     cbuf.set_insts_mark();
  2650     $$$emit8$primary;
  2651     emit_d32_reloc(cbuf,
  2652                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
  2653                    virtual_call_Relocation::spec(virtual_call_oop_addr),
  2654                    RELOC_DISP32);
  2655   %}
  2657   enc_class Java_Compiled_Call(method meth)
  2658   %{
  2659     // JAVA COMPILED CALL
  2660     int disp = in_bytes(methodOopDesc:: from_compiled_offset());
  2662     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
  2663     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
  2665     // callq *disp(%rax)
  2666     cbuf.set_insts_mark();
  2667     $$$emit8$primary;
  2668     if (disp < 0x80) {
  2669       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
  2670       emit_d8(cbuf, disp); // Displacement
  2671     } else {
  2672       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
  2673       emit_d32(cbuf, disp); // Displacement
  2675   %}
  2677   enc_class reg_opc_imm(rRegI dst, immI8 shift)
  2678   %{
  2679     // SAL, SAR, SHR
  2680     int dstenc = $dst$$reg;
  2681     if (dstenc >= 8) {
  2682       emit_opcode(cbuf, Assembler::REX_B);
  2683       dstenc -= 8;
  2685     $$$emit8$primary;
  2686     emit_rm(cbuf, 0x3, $secondary, dstenc);
  2687     $$$emit8$shift$$constant;
  2688   %}
  2690   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
  2691   %{
  2692     // SAL, SAR, SHR
  2693     int dstenc = $dst$$reg;
  2694     if (dstenc < 8) {
  2695       emit_opcode(cbuf, Assembler::REX_W);
  2696     } else {
  2697       emit_opcode(cbuf, Assembler::REX_WB);
  2698       dstenc -= 8;
  2700     $$$emit8$primary;
  2701     emit_rm(cbuf, 0x3, $secondary, dstenc);
  2702     $$$emit8$shift$$constant;
  2703   %}
  2705   enc_class load_immI(rRegI dst, immI src)
  2706   %{
  2707     int dstenc = $dst$$reg;
  2708     if (dstenc >= 8) {
  2709       emit_opcode(cbuf, Assembler::REX_B);
  2710       dstenc -= 8;
  2712     emit_opcode(cbuf, 0xB8 | dstenc);
  2713     $$$emit32$src$$constant;
  2714   %}
  2716   enc_class load_immL(rRegL dst, immL src)
  2717   %{
  2718     int dstenc = $dst$$reg;
  2719     if (dstenc < 8) {
  2720       emit_opcode(cbuf, Assembler::REX_W);
  2721     } else {
  2722       emit_opcode(cbuf, Assembler::REX_WB);
  2723       dstenc -= 8;
  2725     emit_opcode(cbuf, 0xB8 | dstenc);
  2726     emit_d64(cbuf, $src$$constant);
  2727   %}
  2729   enc_class load_immUL32(rRegL dst, immUL32 src)
  2730   %{
  2731     // same as load_immI, but this time we care about zeroes in the high word
  2732     int dstenc = $dst$$reg;
  2733     if (dstenc >= 8) {
  2734       emit_opcode(cbuf, Assembler::REX_B);
  2735       dstenc -= 8;
  2737     emit_opcode(cbuf, 0xB8 | dstenc);
  2738     $$$emit32$src$$constant;
  2739   %}
  2741   enc_class load_immL32(rRegL dst, immL32 src)
  2742   %{
  2743     int dstenc = $dst$$reg;
  2744     if (dstenc < 8) {
  2745       emit_opcode(cbuf, Assembler::REX_W);
  2746     } else {
  2747       emit_opcode(cbuf, Assembler::REX_WB);
  2748       dstenc -= 8;
  2750     emit_opcode(cbuf, 0xC7);
  2751     emit_rm(cbuf, 0x03, 0x00, dstenc);
  2752     $$$emit32$src$$constant;
  2753   %}
  2755   enc_class load_immP31(rRegP dst, immP32 src)
  2756   %{
  2757     // same as load_immI, but this time we care about zeroes in the high word
  2758     int dstenc = $dst$$reg;
  2759     if (dstenc >= 8) {
  2760       emit_opcode(cbuf, Assembler::REX_B);
  2761       dstenc -= 8;
  2763     emit_opcode(cbuf, 0xB8 | dstenc);
  2764     $$$emit32$src$$constant;
  2765   %}
  2767   enc_class load_immP(rRegP dst, immP src)
  2768   %{
  2769     int dstenc = $dst$$reg;
  2770     if (dstenc < 8) {
  2771       emit_opcode(cbuf, Assembler::REX_W);
  2772     } else {
  2773       emit_opcode(cbuf, Assembler::REX_WB);
  2774       dstenc -= 8;
  2776     emit_opcode(cbuf, 0xB8 | dstenc);
  2777     // This next line should be generated from ADLC
  2778     if ($src->constant_is_oop()) {
  2779       emit_d64_reloc(cbuf, $src$$constant, relocInfo::oop_type, RELOC_IMM64);
  2780     } else {
  2781       emit_d64(cbuf, $src$$constant);
  2783   %}
  2785   enc_class load_immF(regF dst, immF con)
  2786   %{
  2787     // XXX reg_mem doesn't support RIP-relative addressing yet
  2788     emit_rm(cbuf, 0x0, $dst$$reg & 7, 0x5); // 00 reg 101
  2789     emit_float_constant(cbuf, $con$$constant);
  2790   %}
  2792   enc_class load_immD(regD dst, immD con)
  2793   %{
  2794     // XXX reg_mem doesn't support RIP-relative addressing yet
  2795     emit_rm(cbuf, 0x0, $dst$$reg & 7, 0x5); // 00 reg 101
  2796     emit_double_constant(cbuf, $con$$constant);
  2797   %}
  2799   enc_class load_conF (regF dst, immF con) %{    // Load float constant
  2800     emit_opcode(cbuf, 0xF3);
  2801     if ($dst$$reg >= 8) {
  2802       emit_opcode(cbuf, Assembler::REX_R);
  2804     emit_opcode(cbuf, 0x0F);
  2805     emit_opcode(cbuf, 0x10);
  2806     emit_rm(cbuf, 0x0, $dst$$reg & 7, 0x5); // 00 reg 101
  2807     emit_float_constant(cbuf, $con$$constant);
  2808   %}
  2810   enc_class load_conD (regD dst, immD con) %{    // Load double constant
  2811     // UseXmmLoadAndClearUpper ? movsd(dst, con) : movlpd(dst, con)
  2812     emit_opcode(cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
  2813     if ($dst$$reg >= 8) {
  2814       emit_opcode(cbuf, Assembler::REX_R);
  2816     emit_opcode(cbuf, 0x0F);
  2817     emit_opcode(cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);
  2818     emit_rm(cbuf, 0x0, $dst$$reg & 7, 0x5); // 00 reg 101
  2819     emit_double_constant(cbuf, $con$$constant);
  2820   %}
  2822   // Encode a reg-reg copy.  If it is useless, then empty encoding.
  2823   enc_class enc_copy(rRegI dst, rRegI src)
  2824   %{
  2825     encode_copy(cbuf, $dst$$reg, $src$$reg);
  2826   %}
  2828   // Encode xmm reg-reg copy.  If it is useless, then empty encoding.
  2829   enc_class enc_CopyXD( RegD dst, RegD src ) %{
  2830     encode_CopyXD( cbuf, $dst$$reg, $src$$reg );
  2831   %}
  2833   enc_class enc_copy_always(rRegI dst, rRegI src)
  2834   %{
  2835     int srcenc = $src$$reg;
  2836     int dstenc = $dst$$reg;
  2838     if (dstenc < 8) {
  2839       if (srcenc >= 8) {
  2840         emit_opcode(cbuf, Assembler::REX_B);
  2841         srcenc -= 8;
  2843     } else {
  2844       if (srcenc < 8) {
  2845         emit_opcode(cbuf, Assembler::REX_R);
  2846       } else {
  2847         emit_opcode(cbuf, Assembler::REX_RB);
  2848         srcenc -= 8;
  2850       dstenc -= 8;
  2853     emit_opcode(cbuf, 0x8B);
  2854     emit_rm(cbuf, 0x3, dstenc, srcenc);
  2855   %}
  2857   enc_class enc_copy_wide(rRegL dst, rRegL src)
  2858   %{
  2859     int srcenc = $src$$reg;
  2860     int dstenc = $dst$$reg;
  2862     if (dstenc != srcenc) {
  2863       if (dstenc < 8) {
  2864         if (srcenc < 8) {
  2865           emit_opcode(cbuf, Assembler::REX_W);
  2866         } else {
  2867           emit_opcode(cbuf, Assembler::REX_WB);
  2868           srcenc -= 8;
  2870       } else {
  2871         if (srcenc < 8) {
  2872           emit_opcode(cbuf, Assembler::REX_WR);
  2873         } else {
  2874           emit_opcode(cbuf, Assembler::REX_WRB);
  2875           srcenc -= 8;
  2877         dstenc -= 8;
  2879       emit_opcode(cbuf, 0x8B);
  2880       emit_rm(cbuf, 0x3, dstenc, srcenc);
  2882   %}
  2884   enc_class Con32(immI src)
  2885   %{
  2886     // Output immediate
  2887     $$$emit32$src$$constant;
  2888   %}
  2890   enc_class Con64(immL src)
  2891   %{
  2892     // Output immediate
  2893     emit_d64($src$$constant);
  2894   %}
  2896   enc_class Con32F_as_bits(immF src)
  2897   %{
  2898     // Output Float immediate bits
  2899     jfloat jf = $src$$constant;
  2900     jint jf_as_bits = jint_cast(jf);
  2901     emit_d32(cbuf, jf_as_bits);
  2902   %}
  2904   enc_class Con16(immI src)
  2905   %{
  2906     // Output immediate
  2907     $$$emit16$src$$constant;
  2908   %}
  2910   // How is this different from Con32??? XXX
  2911   enc_class Con_d32(immI src)
  2912   %{
  2913     emit_d32(cbuf,$src$$constant);
  2914   %}
  2916   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
  2917     // Output immediate memory reference
  2918     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
  2919     emit_d32(cbuf, 0x00);
  2920   %}
  2922   enc_class jump_enc(rRegL switch_val, rRegI dest) %{
  2923     MacroAssembler masm(&cbuf);
  2925     Register switch_reg = as_Register($switch_val$$reg);
  2926     Register dest_reg   = as_Register($dest$$reg);
  2927     address table_base  = masm.address_table_constant(_index2label);
  2929     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
  2930     // to do that and the compiler is using that register as one it can allocate.
  2931     // So we build it all by hand.
  2932     // Address index(noreg, switch_reg, Address::times_1);
  2933     // ArrayAddress dispatch(table, index);
  2935     Address dispatch(dest_reg, switch_reg, Address::times_1);
  2937     masm.lea(dest_reg, InternalAddress(table_base));
  2938     masm.jmp(dispatch);
  2939   %}
  2941   enc_class jump_enc_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
  2942     MacroAssembler masm(&cbuf);
  2944     Register switch_reg = as_Register($switch_val$$reg);
  2945     Register dest_reg   = as_Register($dest$$reg);
  2946     address table_base  = masm.address_table_constant(_index2label);
  2948     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
  2949     // to do that and the compiler is using that register as one it can allocate.
  2950     // So we build it all by hand.
  2951     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant, (int)$offset$$constant);
  2952     // ArrayAddress dispatch(table, index);
  2954     Address dispatch(dest_reg, switch_reg, (Address::ScaleFactor)$shift$$constant, (int)$offset$$constant);
  2956     masm.lea(dest_reg, InternalAddress(table_base));
  2957     masm.jmp(dispatch);
  2958   %}
  2960   enc_class jump_enc_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
  2961     MacroAssembler masm(&cbuf);
  2963     Register switch_reg = as_Register($switch_val$$reg);
  2964     Register dest_reg   = as_Register($dest$$reg);
  2965     address table_base  = masm.address_table_constant(_index2label);
  2967     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
  2968     // to do that and the compiler is using that register as one it can allocate.
  2969     // So we build it all by hand.
  2970     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
  2971     // ArrayAddress dispatch(table, index);
  2973     Address dispatch(dest_reg, switch_reg, (Address::ScaleFactor)$shift$$constant);
  2974     masm.lea(dest_reg, InternalAddress(table_base));
  2975     masm.jmp(dispatch);
  2977   %}
  2979   enc_class lock_prefix()
  2980   %{
  2981     if (os::is_MP()) {
  2982       emit_opcode(cbuf, 0xF0); // lock
  2984   %}
  2986   enc_class REX_mem(memory mem)
  2987   %{
  2988     if ($mem$$base >= 8) {
  2989       if ($mem$$index < 8) {
  2990         emit_opcode(cbuf, Assembler::REX_B);
  2991       } else {
  2992         emit_opcode(cbuf, Assembler::REX_XB);
  2994     } else {
  2995       if ($mem$$index >= 8) {
  2996         emit_opcode(cbuf, Assembler::REX_X);
  2999   %}
  3001   enc_class REX_mem_wide(memory mem)
  3002   %{
  3003     if ($mem$$base >= 8) {
  3004       if ($mem$$index < 8) {
  3005         emit_opcode(cbuf, Assembler::REX_WB);
  3006       } else {
  3007         emit_opcode(cbuf, Assembler::REX_WXB);
  3009     } else {
  3010       if ($mem$$index < 8) {
  3011         emit_opcode(cbuf, Assembler::REX_W);
  3012       } else {
  3013         emit_opcode(cbuf, Assembler::REX_WX);
  3016   %}
  3018   // for byte regs
  3019   enc_class REX_breg(rRegI reg)
  3020   %{
  3021     if ($reg$$reg >= 4) {
  3022       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
  3024   %}
  3026   // for byte regs
  3027   enc_class REX_reg_breg(rRegI dst, rRegI src)
  3028   %{
  3029     if ($dst$$reg < 8) {
  3030       if ($src$$reg >= 4) {
  3031         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
  3033     } else {
  3034       if ($src$$reg < 8) {
  3035         emit_opcode(cbuf, Assembler::REX_R);
  3036       } else {
  3037         emit_opcode(cbuf, Assembler::REX_RB);
  3040   %}
  3042   // for byte regs
  3043   enc_class REX_breg_mem(rRegI reg, memory mem)
  3044   %{
  3045     if ($reg$$reg < 8) {
  3046       if ($mem$$base < 8) {
  3047         if ($mem$$index >= 8) {
  3048           emit_opcode(cbuf, Assembler::REX_X);
  3049         } else if ($reg$$reg >= 4) {
  3050           emit_opcode(cbuf, Assembler::REX);
  3052       } else {
  3053         if ($mem$$index < 8) {
  3054           emit_opcode(cbuf, Assembler::REX_B);
  3055         } else {
  3056           emit_opcode(cbuf, Assembler::REX_XB);
  3059     } else {
  3060       if ($mem$$base < 8) {
  3061         if ($mem$$index < 8) {
  3062           emit_opcode(cbuf, Assembler::REX_R);
  3063         } else {
  3064           emit_opcode(cbuf, Assembler::REX_RX);
  3066       } else {
  3067         if ($mem$$index < 8) {
  3068           emit_opcode(cbuf, Assembler::REX_RB);
  3069         } else {
  3070           emit_opcode(cbuf, Assembler::REX_RXB);
  3074   %}
  3076   enc_class REX_reg(rRegI reg)
  3077   %{
  3078     if ($reg$$reg >= 8) {
  3079       emit_opcode(cbuf, Assembler::REX_B);
  3081   %}
  3083   enc_class REX_reg_wide(rRegI reg)
  3084   %{
  3085     if ($reg$$reg < 8) {
  3086       emit_opcode(cbuf, Assembler::REX_W);
  3087     } else {
  3088       emit_opcode(cbuf, Assembler::REX_WB);
  3090   %}
  3092   enc_class REX_reg_reg(rRegI dst, rRegI src)
  3093   %{
  3094     if ($dst$$reg < 8) {
  3095       if ($src$$reg >= 8) {
  3096         emit_opcode(cbuf, Assembler::REX_B);
  3098     } else {
  3099       if ($src$$reg < 8) {
  3100         emit_opcode(cbuf, Assembler::REX_R);
  3101       } else {
  3102         emit_opcode(cbuf, Assembler::REX_RB);
  3105   %}
  3107   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
  3108   %{
  3109     if ($dst$$reg < 8) {
  3110       if ($src$$reg < 8) {
  3111         emit_opcode(cbuf, Assembler::REX_W);
  3112       } else {
  3113         emit_opcode(cbuf, Assembler::REX_WB);
  3115     } else {
  3116       if ($src$$reg < 8) {
  3117         emit_opcode(cbuf, Assembler::REX_WR);
  3118       } else {
  3119         emit_opcode(cbuf, Assembler::REX_WRB);
  3122   %}
  3124   enc_class REX_reg_mem(rRegI reg, memory mem)
  3125   %{
  3126     if ($reg$$reg < 8) {
  3127       if ($mem$$base < 8) {
  3128         if ($mem$$index >= 8) {
  3129           emit_opcode(cbuf, Assembler::REX_X);
  3131       } else {
  3132         if ($mem$$index < 8) {
  3133           emit_opcode(cbuf, Assembler::REX_B);
  3134         } else {
  3135           emit_opcode(cbuf, Assembler::REX_XB);
  3138     } else {
  3139       if ($mem$$base < 8) {
  3140         if ($mem$$index < 8) {
  3141           emit_opcode(cbuf, Assembler::REX_R);
  3142         } else {
  3143           emit_opcode(cbuf, Assembler::REX_RX);
  3145       } else {
  3146         if ($mem$$index < 8) {
  3147           emit_opcode(cbuf, Assembler::REX_RB);
  3148         } else {
  3149           emit_opcode(cbuf, Assembler::REX_RXB);
  3153   %}
  3155   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
  3156   %{
  3157     if ($reg$$reg < 8) {
  3158       if ($mem$$base < 8) {
  3159         if ($mem$$index < 8) {
  3160           emit_opcode(cbuf, Assembler::REX_W);
  3161         } else {
  3162           emit_opcode(cbuf, Assembler::REX_WX);
  3164       } else {
  3165         if ($mem$$index < 8) {
  3166           emit_opcode(cbuf, Assembler::REX_WB);
  3167         } else {
  3168           emit_opcode(cbuf, Assembler::REX_WXB);
  3171     } else {
  3172       if ($mem$$base < 8) {
  3173         if ($mem$$index < 8) {
  3174           emit_opcode(cbuf, Assembler::REX_WR);
  3175         } else {
  3176           emit_opcode(cbuf, Assembler::REX_WRX);
  3178       } else {
  3179         if ($mem$$index < 8) {
  3180           emit_opcode(cbuf, Assembler::REX_WRB);
  3181         } else {
  3182           emit_opcode(cbuf, Assembler::REX_WRXB);
  3186   %}
  3188   enc_class reg_mem(rRegI ereg, memory mem)
  3189   %{
  3190     // High registers handle in encode_RegMem
  3191     int reg = $ereg$$reg;
  3192     int base = $mem$$base;
  3193     int index = $mem$$index;
  3194     int scale = $mem$$scale;
  3195     int disp = $mem$$disp;
  3196     bool disp_is_oop = $mem->disp_is_oop();
  3198     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_is_oop);
  3199   %}
  3201   enc_class RM_opc_mem(immI rm_opcode, memory mem)
  3202   %{
  3203     int rm_byte_opcode = $rm_opcode$$constant;
  3205     // High registers handle in encode_RegMem
  3206     int base = $mem$$base;
  3207     int index = $mem$$index;
  3208     int scale = $mem$$scale;
  3209     int displace = $mem$$disp;
  3211     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when
  3212                                             // working with static
  3213                                             // globals
  3214     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
  3215                   disp_is_oop);
  3216   %}
  3218   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
  3219   %{
  3220     int reg_encoding = $dst$$reg;
  3221     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
  3222     int index        = 0x04;            // 0x04 indicates no index
  3223     int scale        = 0x00;            // 0x00 indicates no scale
  3224     int displace     = $src1$$constant; // 0x00 indicates no displacement
  3225     bool disp_is_oop = false;
  3226     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
  3227                   disp_is_oop);
  3228   %}
  3230   enc_class neg_reg(rRegI dst)
  3231   %{
  3232     int dstenc = $dst$$reg;
  3233     if (dstenc >= 8) {
  3234       emit_opcode(cbuf, Assembler::REX_B);
  3235       dstenc -= 8;
  3237     // NEG $dst
  3238     emit_opcode(cbuf, 0xF7);
  3239     emit_rm(cbuf, 0x3, 0x03, dstenc);
  3240   %}
  3242   enc_class neg_reg_wide(rRegI dst)
  3243   %{
  3244     int dstenc = $dst$$reg;
  3245     if (dstenc < 8) {
  3246       emit_opcode(cbuf, Assembler::REX_W);
  3247     } else {
  3248       emit_opcode(cbuf, Assembler::REX_WB);
  3249       dstenc -= 8;
  3251     // NEG $dst
  3252     emit_opcode(cbuf, 0xF7);
  3253     emit_rm(cbuf, 0x3, 0x03, dstenc);
  3254   %}
  3256   enc_class setLT_reg(rRegI dst)
  3257   %{
  3258     int dstenc = $dst$$reg;
  3259     if (dstenc >= 8) {
  3260       emit_opcode(cbuf, Assembler::REX_B);
  3261       dstenc -= 8;
  3262     } else if (dstenc >= 4) {
  3263       emit_opcode(cbuf, Assembler::REX);
  3265     // SETLT $dst
  3266     emit_opcode(cbuf, 0x0F);
  3267     emit_opcode(cbuf, 0x9C);
  3268     emit_rm(cbuf, 0x3, 0x0, dstenc);
  3269   %}
  3271   enc_class setNZ_reg(rRegI dst)
  3272   %{
  3273     int dstenc = $dst$$reg;
  3274     if (dstenc >= 8) {
  3275       emit_opcode(cbuf, Assembler::REX_B);
  3276       dstenc -= 8;
  3277     } else if (dstenc >= 4) {
  3278       emit_opcode(cbuf, Assembler::REX);
  3280     // SETNZ $dst
  3281     emit_opcode(cbuf, 0x0F);
  3282     emit_opcode(cbuf, 0x95);
  3283     emit_rm(cbuf, 0x3, 0x0, dstenc);
  3284   %}
  3286   enc_class enc_cmpLTP(no_rcx_RegI p, no_rcx_RegI q, no_rcx_RegI y,
  3287                        rcx_RegI tmp)
  3288   %{
  3289     // cadd_cmpLT
  3291     int tmpReg = $tmp$$reg;
  3293     int penc = $p$$reg;
  3294     int qenc = $q$$reg;
  3295     int yenc = $y$$reg;
  3297     // subl $p,$q
  3298     if (penc < 8) {
  3299       if (qenc >= 8) {
  3300         emit_opcode(cbuf, Assembler::REX_B);
  3302     } else {
  3303       if (qenc < 8) {
  3304         emit_opcode(cbuf, Assembler::REX_R);
  3305       } else {
  3306         emit_opcode(cbuf, Assembler::REX_RB);
  3309     emit_opcode(cbuf, 0x2B);
  3310     emit_rm(cbuf, 0x3, penc & 7, qenc & 7);
  3312     // sbbl $tmp, $tmp
  3313     emit_opcode(cbuf, 0x1B);
  3314     emit_rm(cbuf, 0x3, tmpReg, tmpReg);
  3316     // andl $tmp, $y
  3317     if (yenc >= 8) {
  3318       emit_opcode(cbuf, Assembler::REX_B);
  3320     emit_opcode(cbuf, 0x23);
  3321     emit_rm(cbuf, 0x3, tmpReg, yenc & 7);
  3323     // addl $p,$tmp
  3324     if (penc >= 8) {
  3325         emit_opcode(cbuf, Assembler::REX_R);
  3327     emit_opcode(cbuf, 0x03);
  3328     emit_rm(cbuf, 0x3, penc & 7, tmpReg);
  3329   %}
  3331   // Compare the lonogs and set -1, 0, or 1 into dst
  3332   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
  3333   %{
  3334     int src1enc = $src1$$reg;
  3335     int src2enc = $src2$$reg;
  3336     int dstenc = $dst$$reg;
  3338     // cmpq $src1, $src2
  3339     if (src1enc < 8) {
  3340       if (src2enc < 8) {
  3341         emit_opcode(cbuf, Assembler::REX_W);
  3342       } else {
  3343         emit_opcode(cbuf, Assembler::REX_WB);
  3345     } else {
  3346       if (src2enc < 8) {
  3347         emit_opcode(cbuf, Assembler::REX_WR);
  3348       } else {
  3349         emit_opcode(cbuf, Assembler::REX_WRB);
  3352     emit_opcode(cbuf, 0x3B);
  3353     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
  3355     // movl $dst, -1
  3356     if (dstenc >= 8) {
  3357       emit_opcode(cbuf, Assembler::REX_B);
  3359     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
  3360     emit_d32(cbuf, -1);
  3362     // jl,s done
  3363     emit_opcode(cbuf, 0x7C);
  3364     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
  3366     // setne $dst
  3367     if (dstenc >= 4) {
  3368       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
  3370     emit_opcode(cbuf, 0x0F);
  3371     emit_opcode(cbuf, 0x95);
  3372     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
  3374     // movzbl $dst, $dst
  3375     if (dstenc >= 4) {
  3376       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
  3378     emit_opcode(cbuf, 0x0F);
  3379     emit_opcode(cbuf, 0xB6);
  3380     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
  3381   %}
  3383   enc_class Push_ResultXD(regD dst) %{
  3384     int dstenc = $dst$$reg;
  3386     store_to_stackslot( cbuf, 0xDD, 0x03, 0 ); //FSTP [RSP]
  3388     // UseXmmLoadAndClearUpper ? movsd dst,[rsp] : movlpd dst,[rsp]
  3389     emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
  3390     if (dstenc >= 8) {
  3391       emit_opcode(cbuf, Assembler::REX_R);
  3393     emit_opcode  (cbuf, 0x0F );
  3394     emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12 );
  3395     encode_RegMem(cbuf, dstenc, RSP_enc, 0x4, 0, 0, false);
  3397     // add rsp,8
  3398     emit_opcode(cbuf, Assembler::REX_W);
  3399     emit_opcode(cbuf,0x83);
  3400     emit_rm(cbuf,0x3, 0x0, RSP_enc);
  3401     emit_d8(cbuf,0x08);
  3402   %}
  3404   enc_class Push_SrcXD(regD src) %{
  3405     int srcenc = $src$$reg;
  3407     // subq rsp,#8
  3408     emit_opcode(cbuf, Assembler::REX_W);
  3409     emit_opcode(cbuf, 0x83);
  3410     emit_rm(cbuf, 0x3, 0x5, RSP_enc);
  3411     emit_d8(cbuf, 0x8);
  3413     // movsd [rsp],src
  3414     emit_opcode(cbuf, 0xF2);
  3415     if (srcenc >= 8) {
  3416       emit_opcode(cbuf, Assembler::REX_R);
  3418     emit_opcode(cbuf, 0x0F);
  3419     emit_opcode(cbuf, 0x11);
  3420     encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false);
  3422     // fldd [rsp]
  3423     emit_opcode(cbuf, 0x66);
  3424     emit_opcode(cbuf, 0xDD);
  3425     encode_RegMem(cbuf, 0x0, RSP_enc, 0x4, 0, 0, false);
  3426   %}
  3429   enc_class movq_ld(regD dst, memory mem) %{
  3430     MacroAssembler _masm(&cbuf);
  3431     __ movq($dst$$XMMRegister, $mem$$Address);
  3432   %}
  3434   enc_class movq_st(memory mem, regD src) %{
  3435     MacroAssembler _masm(&cbuf);
  3436     __ movq($mem$$Address, $src$$XMMRegister);
  3437   %}
  3439   enc_class pshufd_8x8(regF dst, regF src) %{
  3440     MacroAssembler _masm(&cbuf);
  3442     encode_CopyXD(cbuf, $dst$$reg, $src$$reg);
  3443     __ punpcklbw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg));
  3444     __ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg), 0x00);
  3445   %}
  3447   enc_class pshufd_4x16(regF dst, regF src) %{
  3448     MacroAssembler _masm(&cbuf);
  3450     __ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), 0x00);
  3451   %}
  3453   enc_class pshufd(regD dst, regD src, int mode) %{
  3454     MacroAssembler _masm(&cbuf);
  3456     __ pshufd(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), $mode);
  3457   %}
  3459   enc_class pxor(regD dst, regD src) %{
  3460     MacroAssembler _masm(&cbuf);
  3462     __ pxor(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg));
  3463   %}
  3465   enc_class mov_i2x(regD dst, rRegI src) %{
  3466     MacroAssembler _masm(&cbuf);
  3468     __ movdl(as_XMMRegister($dst$$reg), as_Register($src$$reg));
  3469   %}
  3471   // obj: object to lock
  3472   // box: box address (header location) -- killed
  3473   // tmp: rax -- killed
  3474   // scr: rbx -- killed
  3475   //
  3476   // What follows is a direct transliteration of fast_lock() and fast_unlock()
  3477   // from i486.ad.  See that file for comments.
  3478   // TODO: where possible switch from movq (r, 0) to movl(r,0) and
  3479   // use the shorter encoding.  (Movl clears the high-order 32-bits).
  3482   enc_class Fast_Lock(rRegP obj, rRegP box, rax_RegI tmp, rRegP scr)
  3483   %{
  3484     Register objReg = as_Register((int)$obj$$reg);
  3485     Register boxReg = as_Register((int)$box$$reg);
  3486     Register tmpReg = as_Register($tmp$$reg);
  3487     Register scrReg = as_Register($scr$$reg);
  3488     MacroAssembler masm(&cbuf);
  3490     // Verify uniqueness of register assignments -- necessary but not sufficient
  3491     assert (objReg != boxReg && objReg != tmpReg &&
  3492             objReg != scrReg && tmpReg != scrReg, "invariant") ;
  3494     if (_counters != NULL) {
  3495       masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
  3497     if (EmitSync & 1) {
  3498         // Without cast to int32_t a movptr will destroy r10 which is typically obj
  3499         masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ; 
  3500         masm.cmpptr(rsp, (int32_t)NULL_WORD) ; 
  3501     } else
  3502     if (EmitSync & 2) {
  3503         Label DONE_LABEL;
  3504         if (UseBiasedLocking) {
  3505            // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
  3506           masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
  3508         // QQQ was movl...
  3509         masm.movptr(tmpReg, 0x1);
  3510         masm.orptr(tmpReg, Address(objReg, 0));
  3511         masm.movptr(Address(boxReg, 0), tmpReg);
  3512         if (os::is_MP()) {
  3513           masm.lock();
  3515         masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
  3516         masm.jcc(Assembler::equal, DONE_LABEL);
  3518         // Recursive locking
  3519         masm.subptr(tmpReg, rsp);
  3520         masm.andptr(tmpReg, 7 - os::vm_page_size());
  3521         masm.movptr(Address(boxReg, 0), tmpReg);
  3523         masm.bind(DONE_LABEL);
  3524         masm.nop(); // avoid branch to branch
  3525     } else {
  3526         Label DONE_LABEL, IsInflated, Egress;
  3528         masm.movptr(tmpReg, Address(objReg, 0)) ; 
  3529         masm.testl (tmpReg, 0x02) ;         // inflated vs stack-locked|neutral|biased
  3530         masm.jcc   (Assembler::notZero, IsInflated) ; 
  3532         // it's stack-locked, biased or neutral
  3533         // TODO: optimize markword triage order to reduce the number of
  3534         // conditional branches in the most common cases.
  3535         // Beware -- there's a subtle invariant that fetch of the markword
  3536         // at [FETCH], below, will never observe a biased encoding (*101b).
  3537         // If this invariant is not held we'll suffer exclusion (safety) failure.
  3539         if (UseBiasedLocking && !UseOptoBiasInlining) {
  3540           masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters);
  3541           masm.movptr(tmpReg, Address(objReg, 0)) ;        // [FETCH]
  3544         // was q will it destroy high?
  3545         masm.orl   (tmpReg, 1) ; 
  3546         masm.movptr(Address(boxReg, 0), tmpReg) ;  
  3547         if (os::is_MP()) { masm.lock(); } 
  3548         masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
  3549         if (_counters != NULL) {
  3550            masm.cond_inc32(Assembler::equal,
  3551                            ExternalAddress((address) _counters->fast_path_entry_count_addr()));
  3553         masm.jcc   (Assembler::equal, DONE_LABEL);
  3555         // Recursive locking
  3556         masm.subptr(tmpReg, rsp);
  3557         masm.andptr(tmpReg, 7 - os::vm_page_size());
  3558         masm.movptr(Address(boxReg, 0), tmpReg);
  3559         if (_counters != NULL) {
  3560            masm.cond_inc32(Assembler::equal,
  3561                            ExternalAddress((address) _counters->fast_path_entry_count_addr()));
  3563         masm.jmp   (DONE_LABEL) ;
  3565         masm.bind  (IsInflated) ;
  3566         // It's inflated
  3568         // TODO: someday avoid the ST-before-CAS penalty by
  3569         // relocating (deferring) the following ST.
  3570         // We should also think about trying a CAS without having
  3571         // fetched _owner.  If the CAS is successful we may
  3572         // avoid an RTO->RTS upgrade on the $line.
  3573         // Without cast to int32_t a movptr will destroy r10 which is typically obj
  3574         masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ; 
  3576         masm.mov    (boxReg, tmpReg) ; 
  3577         masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 
  3578         masm.testptr(tmpReg, tmpReg) ;   
  3579         masm.jcc    (Assembler::notZero, DONE_LABEL) ; 
  3581         // It's inflated and appears unlocked
  3582         if (os::is_MP()) { masm.lock(); } 
  3583         masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 
  3584         // Intentional fall-through into DONE_LABEL ...
  3586         masm.bind  (DONE_LABEL) ;
  3587         masm.nop   () ;                 // avoid jmp to jmp
  3589   %}
  3591   // obj: object to unlock
  3592   // box: box address (displaced header location), killed
  3593   // RBX: killed tmp; cannot be obj nor box
  3594   enc_class Fast_Unlock(rRegP obj, rax_RegP box, rRegP tmp)
  3595   %{
  3597     Register objReg = as_Register($obj$$reg);
  3598     Register boxReg = as_Register($box$$reg);
  3599     Register tmpReg = as_Register($tmp$$reg);
  3600     MacroAssembler masm(&cbuf);
  3602     if (EmitSync & 4) { 
  3603        masm.cmpptr(rsp, 0) ; 
  3604     } else
  3605     if (EmitSync & 8) {
  3606        Label DONE_LABEL;
  3607        if (UseBiasedLocking) {
  3608          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
  3611        // Check whether the displaced header is 0
  3612        //(=> recursive unlock)
  3613        masm.movptr(tmpReg, Address(boxReg, 0));
  3614        masm.testptr(tmpReg, tmpReg);
  3615        masm.jcc(Assembler::zero, DONE_LABEL);
  3617        // If not recursive lock, reset the header to displaced header
  3618        if (os::is_MP()) {
  3619          masm.lock();
  3621        masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
  3622        masm.bind(DONE_LABEL);
  3623        masm.nop(); // avoid branch to branch
  3624     } else {
  3625        Label DONE_LABEL, Stacked, CheckSucc ;
  3627        if (UseBiasedLocking && !UseOptoBiasInlining) {
  3628          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
  3631        masm.movptr(tmpReg, Address(objReg, 0)) ; 
  3632        masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ; 
  3633        masm.jcc   (Assembler::zero, DONE_LABEL) ; 
  3634        masm.testl (tmpReg, 0x02) ; 
  3635        masm.jcc   (Assembler::zero, Stacked) ; 
  3637        // It's inflated
  3638        masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 
  3639        masm.xorptr(boxReg, r15_thread) ; 
  3640        masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; 
  3641        masm.jcc   (Assembler::notZero, DONE_LABEL) ; 
  3642        masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; 
  3643        masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; 
  3644        masm.jcc   (Assembler::notZero, CheckSucc) ; 
  3645        masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ; 
  3646        masm.jmp   (DONE_LABEL) ; 
  3648        if ((EmitSync & 65536) == 0) { 
  3649          Label LSuccess, LGoSlowPath ;
  3650          masm.bind  (CheckSucc) ;
  3651          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
  3652          masm.jcc   (Assembler::zero, LGoSlowPath) ;
  3654          // I'd much rather use lock:andl m->_owner, 0 as it's faster than the
  3655          // the explicit ST;MEMBAR combination, but masm doesn't currently support
  3656          // "ANDQ M,IMM".  Don't use MFENCE here.  lock:add to TOS, xchg, etc
  3657          // are all faster when the write buffer is populated.
  3658          masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
  3659          if (os::is_MP()) {
  3660             masm.lock () ; masm.addl (Address(rsp, 0), 0) ;
  3662          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
  3663          masm.jcc   (Assembler::notZero, LSuccess) ;
  3665          masm.movptr (boxReg, (int32_t)NULL_WORD) ;                   // box is really EAX
  3666          if (os::is_MP()) { masm.lock(); }
  3667          masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
  3668          masm.jcc   (Assembler::notEqual, LSuccess) ;
  3669          // Intentional fall-through into slow-path
  3671          masm.bind  (LGoSlowPath) ;
  3672          masm.orl   (boxReg, 1) ;                      // set ICC.ZF=0 to indicate failure
  3673          masm.jmp   (DONE_LABEL) ;
  3675          masm.bind  (LSuccess) ;
  3676          masm.testl (boxReg, 0) ;                      // set ICC.ZF=1 to indicate success
  3677          masm.jmp   (DONE_LABEL) ;
  3680        masm.bind  (Stacked) ; 
  3681        masm.movptr(tmpReg, Address (boxReg, 0)) ;      // re-fetch
  3682        if (os::is_MP()) { masm.lock(); } 
  3683        masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
  3685        if (EmitSync & 65536) {
  3686           masm.bind (CheckSucc) ;
  3688        masm.bind(DONE_LABEL);
  3689        if (EmitSync & 32768) {
  3690           masm.nop();                      // avoid branch to branch
  3693   %}
  3696   enc_class enc_rethrow()
  3697   %{
  3698     cbuf.set_insts_mark();
  3699     emit_opcode(cbuf, 0xE9); // jmp entry
  3700     emit_d32_reloc(cbuf,
  3701                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
  3702                    runtime_call_Relocation::spec(),
  3703                    RELOC_DISP32);
  3704   %}
  3706   enc_class absF_encoding(regF dst)
  3707   %{
  3708     int dstenc = $dst$$reg;
  3709     address signmask_address = (address) StubRoutines::x86::float_sign_mask();
  3711     cbuf.set_insts_mark();
  3712     if (dstenc >= 8) {
  3713       emit_opcode(cbuf, Assembler::REX_R);
  3714       dstenc -= 8;
  3716     // XXX reg_mem doesn't support RIP-relative addressing yet
  3717     emit_opcode(cbuf, 0x0F);
  3718     emit_opcode(cbuf, 0x54);
  3719     emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
  3720     emit_d32_reloc(cbuf, signmask_address);
  3721   %}
  3723   enc_class absD_encoding(regD dst)
  3724   %{
  3725     int dstenc = $dst$$reg;
  3726     address signmask_address = (address) StubRoutines::x86::double_sign_mask();
  3728     cbuf.set_insts_mark();
  3729     emit_opcode(cbuf, 0x66);
  3730     if (dstenc >= 8) {
  3731       emit_opcode(cbuf, Assembler::REX_R);
  3732       dstenc -= 8;
  3734     // XXX reg_mem doesn't support RIP-relative addressing yet
  3735     emit_opcode(cbuf, 0x0F);
  3736     emit_opcode(cbuf, 0x54);
  3737     emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
  3738     emit_d32_reloc(cbuf, signmask_address);
  3739   %}
  3741   enc_class negF_encoding(regF dst)
  3742   %{
  3743     int dstenc = $dst$$reg;
  3744     address signflip_address = (address) StubRoutines::x86::float_sign_flip();
  3746     cbuf.set_insts_mark();
  3747     if (dstenc >= 8) {
  3748       emit_opcode(cbuf, Assembler::REX_R);
  3749       dstenc -= 8;
  3751     // XXX reg_mem doesn't support RIP-relative addressing yet
  3752     emit_opcode(cbuf, 0x0F);
  3753     emit_opcode(cbuf, 0x57);
  3754     emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
  3755     emit_d32_reloc(cbuf, signflip_address);
  3756   %}
  3758   enc_class negD_encoding(regD dst)
  3759   %{
  3760     int dstenc = $dst$$reg;
  3761     address signflip_address = (address) StubRoutines::x86::double_sign_flip();
  3763     cbuf.set_insts_mark();
  3764     emit_opcode(cbuf, 0x66);
  3765     if (dstenc >= 8) {
  3766       emit_opcode(cbuf, Assembler::REX_R);
  3767       dstenc -= 8;
  3769     // XXX reg_mem doesn't support RIP-relative addressing yet
  3770     emit_opcode(cbuf, 0x0F);
  3771     emit_opcode(cbuf, 0x57);
  3772     emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
  3773     emit_d32_reloc(cbuf, signflip_address);
  3774   %}
  3776   enc_class f2i_fixup(rRegI dst, regF src)
  3777   %{
  3778     int dstenc = $dst$$reg;
  3779     int srcenc = $src$$reg;
  3781     // cmpl $dst, #0x80000000
  3782     if (dstenc >= 8) {
  3783       emit_opcode(cbuf, Assembler::REX_B);
  3785     emit_opcode(cbuf, 0x81);
  3786     emit_rm(cbuf, 0x3, 0x7, dstenc & 7);
  3787     emit_d32(cbuf, 0x80000000);
  3789     // jne,s done
  3790     emit_opcode(cbuf, 0x75);
  3791     if (srcenc < 8 && dstenc < 8) {
  3792       emit_d8(cbuf, 0xF);
  3793     } else if (srcenc >= 8 && dstenc >= 8) {
  3794       emit_d8(cbuf, 0x11);
  3795     } else {
  3796       emit_d8(cbuf, 0x10);
  3799     // subq rsp, #8
  3800     emit_opcode(cbuf, Assembler::REX_W);
  3801     emit_opcode(cbuf, 0x83);
  3802     emit_rm(cbuf, 0x3, 0x5, RSP_enc);
  3803     emit_d8(cbuf, 8);
  3805     // movss [rsp], $src
  3806     emit_opcode(cbuf, 0xF3);
  3807     if (srcenc >= 8) {
  3808       emit_opcode(cbuf, Assembler::REX_R);
  3810     emit_opcode(cbuf, 0x0F);
  3811     emit_opcode(cbuf, 0x11);
  3812     encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
  3814     // call f2i_fixup
  3815     cbuf.set_insts_mark();
  3816     emit_opcode(cbuf, 0xE8);
  3817     emit_d32_reloc(cbuf,
  3818                    (int)
  3819                    (StubRoutines::x86::f2i_fixup() - cbuf.insts_end() - 4),
  3820                    runtime_call_Relocation::spec(),
  3821                    RELOC_DISP32);
  3823     // popq $dst
  3824     if (dstenc >= 8) {
  3825       emit_opcode(cbuf, Assembler::REX_B);
  3827     emit_opcode(cbuf, 0x58 | (dstenc & 7));
  3829     // done:
  3830   %}
  3832   enc_class f2l_fixup(rRegL dst, regF src)
  3833   %{
  3834     int dstenc = $dst$$reg;
  3835     int srcenc = $src$$reg;
  3836     address const_address = (address) StubRoutines::x86::double_sign_flip();
  3838     // cmpq $dst, [0x8000000000000000]
  3839     cbuf.set_insts_mark();
  3840     emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR);
  3841     emit_opcode(cbuf, 0x39);
  3842     // XXX reg_mem doesn't support RIP-relative addressing yet
  3843     emit_rm(cbuf, 0x0, dstenc & 7, 0x5); // 00 reg 101
  3844     emit_d32_reloc(cbuf, const_address);
  3847     // jne,s done
  3848     emit_opcode(cbuf, 0x75);
  3849     if (srcenc < 8 && dstenc < 8) {
  3850       emit_d8(cbuf, 0xF);
  3851     } else if (srcenc >= 8 && dstenc >= 8) {
  3852       emit_d8(cbuf, 0x11);
  3853     } else {
  3854       emit_d8(cbuf, 0x10);
  3857     // subq rsp, #8
  3858     emit_opcode(cbuf, Assembler::REX_W);
  3859     emit_opcode(cbuf, 0x83);
  3860     emit_rm(cbuf, 0x3, 0x5, RSP_enc);
  3861     emit_d8(cbuf, 8);
  3863     // movss [rsp], $src
  3864     emit_opcode(cbuf, 0xF3);
  3865     if (srcenc >= 8) {
  3866       emit_opcode(cbuf, Assembler::REX_R);
  3868     emit_opcode(cbuf, 0x0F);
  3869     emit_opcode(cbuf, 0x11);
  3870     encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
  3872     // call f2l_fixup
  3873     cbuf.set_insts_mark();
  3874     emit_opcode(cbuf, 0xE8);
  3875     emit_d32_reloc(cbuf,
  3876                    (int)
  3877                    (StubRoutines::x86::f2l_fixup() - cbuf.insts_end() - 4),
  3878                    runtime_call_Relocation::spec(),
  3879                    RELOC_DISP32);
  3881     // popq $dst
  3882     if (dstenc >= 8) {
  3883       emit_opcode(cbuf, Assembler::REX_B);
  3885     emit_opcode(cbuf, 0x58 | (dstenc & 7));
  3887     // done:
  3888   %}
  3890   enc_class d2i_fixup(rRegI dst, regD src)
  3891   %{
  3892     int dstenc = $dst$$reg;
  3893     int srcenc = $src$$reg;
  3895     // cmpl $dst, #0x80000000
  3896     if (dstenc >= 8) {
  3897       emit_opcode(cbuf, Assembler::REX_B);
  3899     emit_opcode(cbuf, 0x81);
  3900     emit_rm(cbuf, 0x3, 0x7, dstenc & 7);
  3901     emit_d32(cbuf, 0x80000000);
  3903     // jne,s done
  3904     emit_opcode(cbuf, 0x75);
  3905     if (srcenc < 8 && dstenc < 8) {
  3906       emit_d8(cbuf, 0xF);
  3907     } else if (srcenc >= 8 && dstenc >= 8) {
  3908       emit_d8(cbuf, 0x11);
  3909     } else {
  3910       emit_d8(cbuf, 0x10);
  3913     // subq rsp, #8
  3914     emit_opcode(cbuf, Assembler::REX_W);
  3915     emit_opcode(cbuf, 0x83);
  3916     emit_rm(cbuf, 0x3, 0x5, RSP_enc);
  3917     emit_d8(cbuf, 8);
  3919     // movsd [rsp], $src
  3920     emit_opcode(cbuf, 0xF2);
  3921     if (srcenc >= 8) {
  3922       emit_opcode(cbuf, Assembler::REX_R);
  3924     emit_opcode(cbuf, 0x0F);
  3925     emit_opcode(cbuf, 0x11);
  3926     encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
  3928     // call d2i_fixup
  3929     cbuf.set_insts_mark();
  3930     emit_opcode(cbuf, 0xE8);
  3931     emit_d32_reloc(cbuf,
  3932                    (int)
  3933                    (StubRoutines::x86::d2i_fixup() - cbuf.insts_end() - 4),
  3934                    runtime_call_Relocation::spec(),
  3935                    RELOC_DISP32);
  3937     // popq $dst
  3938     if (dstenc >= 8) {
  3939       emit_opcode(cbuf, Assembler::REX_B);
  3941     emit_opcode(cbuf, 0x58 | (dstenc & 7));
  3943     // done:
  3944   %}
  3946   enc_class d2l_fixup(rRegL dst, regD src)
  3947   %{
  3948     int dstenc = $dst$$reg;
  3949     int srcenc = $src$$reg;
  3950     address const_address = (address) StubRoutines::x86::double_sign_flip();
  3952     // cmpq $dst, [0x8000000000000000]
  3953     cbuf.set_insts_mark();
  3954     emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR);
  3955     emit_opcode(cbuf, 0x39);
  3956     // XXX reg_mem doesn't support RIP-relative addressing yet
  3957     emit_rm(cbuf, 0x0, dstenc & 7, 0x5); // 00 reg 101
  3958     emit_d32_reloc(cbuf, const_address);
  3961     // jne,s done
  3962     emit_opcode(cbuf, 0x75);
  3963     if (srcenc < 8 && dstenc < 8) {
  3964       emit_d8(cbuf, 0xF);
  3965     } else if (srcenc >= 8 && dstenc >= 8) {
  3966       emit_d8(cbuf, 0x11);
  3967     } else {
  3968       emit_d8(cbuf, 0x10);
  3971     // subq rsp, #8
  3972     emit_opcode(cbuf, Assembler::REX_W);
  3973     emit_opcode(cbuf, 0x83);
  3974     emit_rm(cbuf, 0x3, 0x5, RSP_enc);
  3975     emit_d8(cbuf, 8);
  3977     // movsd [rsp], $src
  3978     emit_opcode(cbuf, 0xF2);
  3979     if (srcenc >= 8) {
  3980       emit_opcode(cbuf, Assembler::REX_R);
  3982     emit_opcode(cbuf, 0x0F);
  3983     emit_opcode(cbuf, 0x11);
  3984     encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
  3986     // call d2l_fixup
  3987     cbuf.set_insts_mark();
  3988     emit_opcode(cbuf, 0xE8);
  3989     emit_d32_reloc(cbuf,
  3990                    (int)
  3991                    (StubRoutines::x86::d2l_fixup() - cbuf.insts_end() - 4),
  3992                    runtime_call_Relocation::spec(),
  3993                    RELOC_DISP32);
  3995     // popq $dst
  3996     if (dstenc >= 8) {
  3997       emit_opcode(cbuf, Assembler::REX_B);
  3999     emit_opcode(cbuf, 0x58 | (dstenc & 7));
  4001     // done:
  4002   %}
  4004   // Safepoint Poll.  This polls the safepoint page, and causes an
  4005   // exception if it is not readable. Unfortunately, it kills
  4006   // RFLAGS in the process.
  4007   enc_class enc_safepoint_poll
  4008   %{
  4009     // testl %rax, off(%rip) // Opcode + ModRM + Disp32 == 6 bytes
  4010     // XXX reg_mem doesn't support RIP-relative addressing yet
  4011     cbuf.set_insts_mark();
  4012     cbuf.relocate(cbuf.insts_mark(), relocInfo::poll_type, 0); // XXX
  4013     emit_opcode(cbuf, 0x85); // testl
  4014     emit_rm(cbuf, 0x0, RAX_enc, 0x5); // 00 rax 101 == 0x5
  4015     // cbuf.insts_mark() is beginning of instruction
  4016     emit_d32_reloc(cbuf, os::get_polling_page());
  4017 //                    relocInfo::poll_type,
  4018   %}
  4019 %}
  4023 //----------FRAME--------------------------------------------------------------
  4024 // Definition of frame structure and management information.
  4025 //
  4026 //  S T A C K   L A Y O U T    Allocators stack-slot number
  4027 //                             |   (to get allocators register number
  4028 //  G  Owned by    |        |  v    add OptoReg::stack0())
  4029 //  r   CALLER     |        |
  4030 //  o     |        +--------+      pad to even-align allocators stack-slot
  4031 //  w     V        |  pad0  |        numbers; owned by CALLER
  4032 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
  4033 //  h     ^        |   in   |  5
  4034 //        |        |  args  |  4   Holes in incoming args owned by SELF
  4035 //  |     |        |        |  3
  4036 //  |     |        +--------+
  4037 //  V     |        | old out|      Empty on Intel, window on Sparc
  4038 //        |    old |preserve|      Must be even aligned.
  4039 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
  4040 //        |        |   in   |  3   area for Intel ret address
  4041 //     Owned by    |preserve|      Empty on Sparc.
  4042 //       SELF      +--------+
  4043 //        |        |  pad2  |  2   pad to align old SP
  4044 //        |        +--------+  1
  4045 //        |        | locks  |  0
  4046 //        |        +--------+----> OptoReg::stack0(), even aligned
  4047 //        |        |  pad1  | 11   pad to align new SP
  4048 //        |        +--------+
  4049 //        |        |        | 10
  4050 //        |        | spills |  9   spills
  4051 //        V        |        |  8   (pad0 slot for callee)
  4052 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
  4053 //        ^        |  out   |  7
  4054 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
  4055 //     Owned by    +--------+
  4056 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
  4057 //        |    new |preserve|      Must be even-aligned.
  4058 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
  4059 //        |        |        |
  4060 //
  4061 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
  4062 //         known from SELF's arguments and the Java calling convention.
  4063 //         Region 6-7 is determined per call site.
  4064 // Note 2: If the calling convention leaves holes in the incoming argument
  4065 //         area, those holes are owned by SELF.  Holes in the outgoing area
  4066 //         are owned by the CALLEE.  Holes should not be nessecary in the
  4067 //         incoming area, as the Java calling convention is completely under
  4068 //         the control of the AD file.  Doubles can be sorted and packed to
  4069 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
  4070 //         varargs C calling conventions.
  4071 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
  4072 //         even aligned with pad0 as needed.
  4073 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
  4074 //         region 6-11 is even aligned; it may be padded out more so that
  4075 //         the region from SP to FP meets the minimum stack alignment.
  4076 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
  4077 //         alignment.  Region 11, pad1, may be dynamically extended so that
  4078 //         SP meets the minimum alignment.
  4080 frame
  4081 %{
  4082   // What direction does stack grow in (assumed to be same for C & Java)
  4083   stack_direction(TOWARDS_LOW);
  4085   // These three registers define part of the calling convention
  4086   // between compiled code and the interpreter.
  4087   inline_cache_reg(RAX);                // Inline Cache Register
  4088   interpreter_method_oop_reg(RBX);      // Method Oop Register when
  4089                                         // calling interpreter
  4091   // Optional: name the operand used by cisc-spilling to access
  4092   // [stack_pointer + offset]
  4093   cisc_spilling_operand_name(indOffset32);
  4095   // Number of stack slots consumed by locking an object
  4096   sync_stack_slots(2);
  4098   // Compiled code's Frame Pointer
  4099   frame_pointer(RSP);
  4101   // Interpreter stores its frame pointer in a register which is
  4102   // stored to the stack by I2CAdaptors.
  4103   // I2CAdaptors convert from interpreted java to compiled java.
  4104   interpreter_frame_pointer(RBP);
  4106   // Stack alignment requirement
  4107   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
  4109   // Number of stack slots between incoming argument block and the start of
  4110   // a new frame.  The PROLOG must add this many slots to the stack.  The
  4111   // EPILOG must remove this many slots.  amd64 needs two slots for
  4112   // return address.
  4113   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
  4115   // Number of outgoing stack slots killed above the out_preserve_stack_slots
  4116   // for calls to C.  Supports the var-args backing area for register parms.
  4117   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
  4119   // The after-PROLOG location of the return address.  Location of
  4120   // return address specifies a type (REG or STACK) and a number
  4121   // representing the register number (i.e. - use a register name) or
  4122   // stack slot.
  4123   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
  4124   // Otherwise, it is above the locks and verification slot and alignment word
  4125   return_addr(STACK - 2 +
  4126               round_to(2 + 2 * VerifyStackAtCalls +
  4127                        Compile::current()->fixed_slots(),
  4128                        WordsPerLong * 2));
  4130   // Body of function which returns an integer array locating
  4131   // arguments either in registers or in stack slots.  Passed an array
  4132   // of ideal registers called "sig" and a "length" count.  Stack-slot
  4133   // offsets are based on outgoing arguments, i.e. a CALLER setting up
  4134   // arguments for a CALLEE.  Incoming stack arguments are
  4135   // automatically biased by the preserve_stack_slots field above.
  4137   calling_convention
  4138   %{
  4139     // No difference between ingoing/outgoing just pass false
  4140     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
  4141   %}
  4143   c_calling_convention
  4144   %{
  4145     // This is obviously always outgoing
  4146     (void) SharedRuntime::c_calling_convention(sig_bt, regs, length);
  4147   %}
  4149   // Location of compiled Java return values.  Same as C for now.
  4150   return_value
  4151   %{
  4152     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
  4153            "only return normal values");
  4155     static const int lo[Op_RegL + 1] = {
  4156       0,
  4157       0,
  4158       RAX_num,  // Op_RegN
  4159       RAX_num,  // Op_RegI
  4160       RAX_num,  // Op_RegP
  4161       XMM0_num, // Op_RegF
  4162       XMM0_num, // Op_RegD
  4163       RAX_num   // Op_RegL
  4164     };
  4165     static const int hi[Op_RegL + 1] = {
  4166       0,
  4167       0,
  4168       OptoReg::Bad, // Op_RegN
  4169       OptoReg::Bad, // Op_RegI
  4170       RAX_H_num,    // Op_RegP
  4171       OptoReg::Bad, // Op_RegF
  4172       XMM0_H_num,   // Op_RegD
  4173       RAX_H_num     // Op_RegL
  4174     };
  4175     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 1, "missing type");
  4176     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
  4177   %}
  4178 %}
  4180 //----------ATTRIBUTES---------------------------------------------------------
  4181 //----------Operand Attributes-------------------------------------------------
  4182 op_attrib op_cost(0);        // Required cost attribute
  4184 //----------Instruction Attributes---------------------------------------------
  4185 ins_attrib ins_cost(100);       // Required cost attribute
  4186 ins_attrib ins_size(8);         // Required size attribute (in bits)
  4187 ins_attrib ins_pc_relative(0);  // Required PC Relative flag
  4188 ins_attrib ins_short_branch(0); // Required flag: is this instruction
  4189                                 // a non-matching short branch variant
  4190                                 // of some long branch?
  4191 ins_attrib ins_alignment(1);    // Required alignment attribute (must
  4192                                 // be a power of 2) specifies the
  4193                                 // alignment that some part of the
  4194                                 // instruction (not necessarily the
  4195                                 // start) requires.  If > 1, a
  4196                                 // compute_padding() function must be
  4197                                 // provided for the instruction
  4199 //----------OPERANDS-----------------------------------------------------------
  4200 // Operand definitions must precede instruction definitions for correct parsing
  4201 // in the ADLC because operands constitute user defined types which are used in
  4202 // instruction definitions.
  4204 //----------Simple Operands----------------------------------------------------
  4205 // Immediate Operands
  4206 // Integer Immediate
  4207 operand immI()
  4208 %{
  4209   match(ConI);
  4211   op_cost(10);
  4212   format %{ %}
  4213   interface(CONST_INTER);
  4214 %}
  4216 // Constant for test vs zero
  4217 operand immI0()
  4218 %{
  4219   predicate(n->get_int() == 0);
  4220   match(ConI);
  4222   op_cost(0);
  4223   format %{ %}
  4224   interface(CONST_INTER);
  4225 %}
  4227 // Constant for increment
  4228 operand immI1()
  4229 %{
  4230   predicate(n->get_int() == 1);
  4231   match(ConI);
  4233   op_cost(0);
  4234   format %{ %}
  4235   interface(CONST_INTER);
  4236 %}
  4238 // Constant for decrement
  4239 operand immI_M1()
  4240 %{
  4241   predicate(n->get_int() == -1);
  4242   match(ConI);
  4244   op_cost(0);
  4245   format %{ %}
  4246   interface(CONST_INTER);
  4247 %}
  4249 // Valid scale values for addressing modes
  4250 operand immI2()
  4251 %{
  4252   predicate(0 <= n->get_int() && (n->get_int() <= 3));
  4253   match(ConI);
  4255   format %{ %}
  4256   interface(CONST_INTER);
  4257 %}
  4259 operand immI8()
  4260 %{
  4261   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
  4262   match(ConI);
  4264   op_cost(5);
  4265   format %{ %}
  4266   interface(CONST_INTER);
  4267 %}
  4269 operand immI16()
  4270 %{
  4271   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
  4272   match(ConI);
  4274   op_cost(10);
  4275   format %{ %}
  4276   interface(CONST_INTER);
  4277 %}
  4279 // Constant for long shifts
  4280 operand immI_32()
  4281 %{
  4282   predicate( n->get_int() == 32 );
  4283   match(ConI);
  4285   op_cost(0);
  4286   format %{ %}
  4287   interface(CONST_INTER);
  4288 %}
  4290 // Constant for long shifts
  4291 operand immI_64()
  4292 %{
  4293   predicate( n->get_int() == 64 );
  4294   match(ConI);
  4296   op_cost(0);
  4297   format %{ %}
  4298   interface(CONST_INTER);
  4299 %}
  4301 // Pointer Immediate
  4302 operand immP()
  4303 %{
  4304   match(ConP);
  4306   op_cost(10);
  4307   format %{ %}
  4308   interface(CONST_INTER);
  4309 %}
  4311 // NULL Pointer Immediate
  4312 operand immP0()
  4313 %{
  4314   predicate(n->get_ptr() == 0);
  4315   match(ConP);
  4317   op_cost(5);
  4318   format %{ %}
  4319   interface(CONST_INTER);
  4320 %}
  4322 // Pointer Immediate
  4323 operand immN() %{
  4324   match(ConN);
  4326   op_cost(10);
  4327   format %{ %}
  4328   interface(CONST_INTER);
  4329 %}
  4331 // NULL Pointer Immediate
  4332 operand immN0() %{
  4333   predicate(n->get_narrowcon() == 0);
  4334   match(ConN);
  4336   op_cost(5);
  4337   format %{ %}
  4338   interface(CONST_INTER);
  4339 %}
  4341 operand immP31()
  4342 %{
  4343   predicate(!n->as_Type()->type()->isa_oopptr()
  4344             && (n->get_ptr() >> 31) == 0);
  4345   match(ConP);
  4347   op_cost(5);
  4348   format %{ %}
  4349   interface(CONST_INTER);
  4350 %}
  4353 // Long Immediate
  4354 operand immL()
  4355 %{
  4356   match(ConL);
  4358   op_cost(20);
  4359   format %{ %}
  4360   interface(CONST_INTER);
  4361 %}
  4363 // Long Immediate 8-bit
  4364 operand immL8()
  4365 %{
  4366   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
  4367   match(ConL);
  4369   op_cost(5);
  4370   format %{ %}
  4371   interface(CONST_INTER);
  4372 %}
  4374 // Long Immediate 32-bit unsigned
  4375 operand immUL32()
  4376 %{
  4377   predicate(n->get_long() == (unsigned int) (n->get_long()));
  4378   match(ConL);
  4380   op_cost(10);
  4381   format %{ %}
  4382   interface(CONST_INTER);
  4383 %}
  4385 // Long Immediate 32-bit signed
  4386 operand immL32()
  4387 %{
  4388   predicate(n->get_long() == (int) (n->get_long()));
  4389   match(ConL);
  4391   op_cost(15);
  4392   format %{ %}
  4393   interface(CONST_INTER);
  4394 %}
  4396 // Long Immediate zero
  4397 operand immL0()
  4398 %{
  4399   predicate(n->get_long() == 0L);
  4400   match(ConL);
  4402   op_cost(10);
  4403   format %{ %}
  4404   interface(CONST_INTER);
  4405 %}
  4407 // Constant for increment
  4408 operand immL1()
  4409 %{
  4410   predicate(n->get_long() == 1);
  4411   match(ConL);
  4413   format %{ %}
  4414   interface(CONST_INTER);
  4415 %}
  4417 // Constant for decrement
  4418 operand immL_M1()
  4419 %{
  4420   predicate(n->get_long() == -1);
  4421   match(ConL);
  4423   format %{ %}
  4424   interface(CONST_INTER);
  4425 %}
  4427 // Long Immediate: the value 10
  4428 operand immL10()
  4429 %{
  4430   predicate(n->get_long() == 10);
  4431   match(ConL);
  4433   format %{ %}
  4434   interface(CONST_INTER);
  4435 %}
  4437 // Long immediate from 0 to 127.
  4438 // Used for a shorter form of long mul by 10.
  4439 operand immL_127()
  4440 %{
  4441   predicate(0 <= n->get_long() && n->get_long() < 0x80);
  4442   match(ConL);
  4444   op_cost(10);
  4445   format %{ %}
  4446   interface(CONST_INTER);
  4447 %}
  4449 // Long Immediate: low 32-bit mask
  4450 operand immL_32bits()
  4451 %{
  4452   predicate(n->get_long() == 0xFFFFFFFFL);
  4453   match(ConL);
  4454   op_cost(20);
  4456   format %{ %}
  4457   interface(CONST_INTER);
  4458 %}
  4460 // Float Immediate zero
  4461 operand immF0()
  4462 %{
  4463   predicate(jint_cast(n->getf()) == 0);
  4464   match(ConF);
  4466   op_cost(5);
  4467   format %{ %}
  4468   interface(CONST_INTER);
  4469 %}
  4471 // Float Immediate
  4472 operand immF()
  4473 %{
  4474   match(ConF);
  4476   op_cost(15);
  4477   format %{ %}
  4478   interface(CONST_INTER);
  4479 %}
  4481 // Double Immediate zero
  4482 operand immD0()
  4483 %{
  4484   predicate(jlong_cast(n->getd()) == 0);
  4485   match(ConD);
  4487   op_cost(5);
  4488   format %{ %}
  4489   interface(CONST_INTER);
  4490 %}
  4492 // Double Immediate
  4493 operand immD()
  4494 %{
  4495   match(ConD);
  4497   op_cost(15);
  4498   format %{ %}
  4499   interface(CONST_INTER);
  4500 %}
  4502 // Immediates for special shifts (sign extend)
  4504 // Constants for increment
  4505 operand immI_16()
  4506 %{
  4507   predicate(n->get_int() == 16);
  4508   match(ConI);
  4510   format %{ %}
  4511   interface(CONST_INTER);
  4512 %}
  4514 operand immI_24()
  4515 %{
  4516   predicate(n->get_int() == 24);
  4517   match(ConI);
  4519   format %{ %}
  4520   interface(CONST_INTER);
  4521 %}
  4523 // Constant for byte-wide masking
  4524 operand immI_255()
  4525 %{
  4526   predicate(n->get_int() == 255);
  4527   match(ConI);
  4529   format %{ %}
  4530   interface(CONST_INTER);
  4531 %}
  4533 // Constant for short-wide masking
  4534 operand immI_65535()
  4535 %{
  4536   predicate(n->get_int() == 65535);
  4537   match(ConI);
  4539   format %{ %}
  4540   interface(CONST_INTER);
  4541 %}
  4543 // Constant for byte-wide masking
  4544 operand immL_255()
  4545 %{
  4546   predicate(n->get_long() == 255);
  4547   match(ConL);
  4549   format %{ %}
  4550   interface(CONST_INTER);
  4551 %}
  4553 // Constant for short-wide masking
  4554 operand immL_65535()
  4555 %{
  4556   predicate(n->get_long() == 65535);
  4557   match(ConL);
  4559   format %{ %}
  4560   interface(CONST_INTER);
  4561 %}
  4563 // Register Operands
  4564 // Integer Register
  4565 operand rRegI()
  4566 %{
  4567   constraint(ALLOC_IN_RC(int_reg));
  4568   match(RegI);
  4570   match(rax_RegI);
  4571   match(rbx_RegI);
  4572   match(rcx_RegI);
  4573   match(rdx_RegI);
  4574   match(rdi_RegI);
  4576   format %{ %}
  4577   interface(REG_INTER);
  4578 %}
  4580 // Special Registers
  4581 operand rax_RegI()
  4582 %{
  4583   constraint(ALLOC_IN_RC(int_rax_reg));
  4584   match(RegI);
  4585   match(rRegI);
  4587   format %{ "RAX" %}
  4588   interface(REG_INTER);
  4589 %}
  4591 // Special Registers
  4592 operand rbx_RegI()
  4593 %{
  4594   constraint(ALLOC_IN_RC(int_rbx_reg));
  4595   match(RegI);
  4596   match(rRegI);
  4598   format %{ "RBX" %}
  4599   interface(REG_INTER);
  4600 %}
  4602 operand rcx_RegI()
  4603 %{
  4604   constraint(ALLOC_IN_RC(int_rcx_reg));
  4605   match(RegI);
  4606   match(rRegI);
  4608   format %{ "RCX" %}
  4609   interface(REG_INTER);
  4610 %}
  4612 operand rdx_RegI()
  4613 %{
  4614   constraint(ALLOC_IN_RC(int_rdx_reg));
  4615   match(RegI);
  4616   match(rRegI);
  4618   format %{ "RDX" %}
  4619   interface(REG_INTER);
  4620 %}
  4622 operand rdi_RegI()
  4623 %{
  4624   constraint(ALLOC_IN_RC(int_rdi_reg));
  4625   match(RegI);
  4626   match(rRegI);
  4628   format %{ "RDI" %}
  4629   interface(REG_INTER);
  4630 %}
  4632 operand no_rcx_RegI()
  4633 %{
  4634   constraint(ALLOC_IN_RC(int_no_rcx_reg));
  4635   match(RegI);
  4636   match(rax_RegI);
  4637   match(rbx_RegI);
  4638   match(rdx_RegI);
  4639   match(rdi_RegI);
  4641   format %{ %}
  4642   interface(REG_INTER);
  4643 %}
  4645 operand no_rax_rdx_RegI()
  4646 %{
  4647   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
  4648   match(RegI);
  4649   match(rbx_RegI);
  4650   match(rcx_RegI);
  4651   match(rdi_RegI);
  4653   format %{ %}
  4654   interface(REG_INTER);
  4655 %}
  4657 // Pointer Register
  4658 operand any_RegP()
  4659 %{
  4660   constraint(ALLOC_IN_RC(any_reg));
  4661   match(RegP);
  4662   match(rax_RegP);
  4663   match(rbx_RegP);
  4664   match(rdi_RegP);
  4665   match(rsi_RegP);
  4666   match(rbp_RegP);
  4667   match(r15_RegP);
  4668   match(rRegP);
  4670   format %{ %}
  4671   interface(REG_INTER);
  4672 %}
  4674 operand rRegP()
  4675 %{
  4676   constraint(ALLOC_IN_RC(ptr_reg));
  4677   match(RegP);
  4678   match(rax_RegP);
  4679   match(rbx_RegP);
  4680   match(rdi_RegP);
  4681   match(rsi_RegP);
  4682   match(rbp_RegP);
  4683   match(r15_RegP);  // See Q&A below about r15_RegP.
  4685   format %{ %}
  4686   interface(REG_INTER);
  4687 %}
  4689 operand rRegN() %{
  4690   constraint(ALLOC_IN_RC(int_reg));
  4691   match(RegN);
  4693   format %{ %}
  4694   interface(REG_INTER);
  4695 %}
  4697 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
  4698 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
  4699 // It's fine for an instruction input which expects rRegP to match a r15_RegP.
  4700 // The output of an instruction is controlled by the allocator, which respects
  4701 // register class masks, not match rules.  Unless an instruction mentions
  4702 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
  4703 // by the allocator as an input.
  4705 operand no_rax_RegP()
  4706 %{
  4707   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
  4708   match(RegP);
  4709   match(rbx_RegP);
  4710   match(rsi_RegP);
  4711   match(rdi_RegP);
  4713   format %{ %}
  4714   interface(REG_INTER);
  4715 %}
  4717 operand no_rbp_RegP()
  4718 %{
  4719   constraint(ALLOC_IN_RC(ptr_no_rbp_reg));
  4720   match(RegP);
  4721   match(rbx_RegP);
  4722   match(rsi_RegP);
  4723   match(rdi_RegP);
  4725   format %{ %}
  4726   interface(REG_INTER);
  4727 %}
  4729 operand no_rax_rbx_RegP()
  4730 %{
  4731   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
  4732   match(RegP);
  4733   match(rsi_RegP);
  4734   match(rdi_RegP);
  4736   format %{ %}
  4737   interface(REG_INTER);
  4738 %}
  4740 // Special Registers
  4741 // Return a pointer value
  4742 operand rax_RegP()
  4743 %{
  4744   constraint(ALLOC_IN_RC(ptr_rax_reg));
  4745   match(RegP);
  4746   match(rRegP);
  4748   format %{ %}
  4749   interface(REG_INTER);
  4750 %}
  4752 // Special Registers
  4753 // Return a compressed pointer value
  4754 operand rax_RegN()
  4755 %{
  4756   constraint(ALLOC_IN_RC(int_rax_reg));
  4757   match(RegN);
  4758   match(rRegN);
  4760   format %{ %}
  4761   interface(REG_INTER);
  4762 %}
  4764 // Used in AtomicAdd
  4765 operand rbx_RegP()
  4766 %{
  4767   constraint(ALLOC_IN_RC(ptr_rbx_reg));
  4768   match(RegP);
  4769   match(rRegP);
  4771   format %{ %}
  4772   interface(REG_INTER);
  4773 %}
  4775 operand rsi_RegP()
  4776 %{
  4777   constraint(ALLOC_IN_RC(ptr_rsi_reg));
  4778   match(RegP);
  4779   match(rRegP);
  4781   format %{ %}
  4782   interface(REG_INTER);
  4783 %}
  4785 // Used in rep stosq
  4786 operand rdi_RegP()
  4787 %{
  4788   constraint(ALLOC_IN_RC(ptr_rdi_reg));
  4789   match(RegP);
  4790   match(rRegP);
  4792   format %{ %}
  4793   interface(REG_INTER);
  4794 %}
  4796 operand rbp_RegP()
  4797 %{
  4798   constraint(ALLOC_IN_RC(ptr_rbp_reg));
  4799   match(RegP);
  4800   match(rRegP);
  4802   format %{ %}
  4803   interface(REG_INTER);
  4804 %}
  4806 operand r15_RegP()
  4807 %{
  4808   constraint(ALLOC_IN_RC(ptr_r15_reg));
  4809   match(RegP);
  4810   match(rRegP);
  4812   format %{ %}
  4813   interface(REG_INTER);
  4814 %}
  4816 operand rRegL()
  4817 %{
  4818   constraint(ALLOC_IN_RC(long_reg));
  4819   match(RegL);
  4820   match(rax_RegL);
  4821   match(rdx_RegL);
  4823   format %{ %}
  4824   interface(REG_INTER);
  4825 %}
  4827 // Special Registers
  4828 operand no_rax_rdx_RegL()
  4829 %{
  4830   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
  4831   match(RegL);
  4832   match(rRegL);
  4834   format %{ %}
  4835   interface(REG_INTER);
  4836 %}
  4838 operand no_rax_RegL()
  4839 %{
  4840   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
  4841   match(RegL);
  4842   match(rRegL);
  4843   match(rdx_RegL);
  4845   format %{ %}
  4846   interface(REG_INTER);
  4847 %}
  4849 operand no_rcx_RegL()
  4850 %{
  4851   constraint(ALLOC_IN_RC(long_no_rcx_reg));
  4852   match(RegL);
  4853   match(rRegL);
  4855   format %{ %}
  4856   interface(REG_INTER);
  4857 %}
  4859 operand rax_RegL()
  4860 %{
  4861   constraint(ALLOC_IN_RC(long_rax_reg));
  4862   match(RegL);
  4863   match(rRegL);
  4865   format %{ "RAX" %}
  4866   interface(REG_INTER);
  4867 %}
  4869 operand rcx_RegL()
  4870 %{
  4871   constraint(ALLOC_IN_RC(long_rcx_reg));
  4872   match(RegL);
  4873   match(rRegL);
  4875   format %{ %}
  4876   interface(REG_INTER);
  4877 %}
  4879 operand rdx_RegL()
  4880 %{
  4881   constraint(ALLOC_IN_RC(long_rdx_reg));
  4882   match(RegL);
  4883   match(rRegL);
  4885   format %{ %}
  4886   interface(REG_INTER);
  4887 %}
  4889 // Flags register, used as output of compare instructions
  4890 operand rFlagsReg()
  4891 %{
  4892   constraint(ALLOC_IN_RC(int_flags));
  4893   match(RegFlags);
  4895   format %{ "RFLAGS" %}
  4896   interface(REG_INTER);
  4897 %}
  4899 // Flags register, used as output of FLOATING POINT compare instructions
  4900 operand rFlagsRegU()
  4901 %{
  4902   constraint(ALLOC_IN_RC(int_flags));
  4903   match(RegFlags);
  4905   format %{ "RFLAGS_U" %}
  4906   interface(REG_INTER);
  4907 %}
  4909 operand rFlagsRegUCF() %{
  4910   constraint(ALLOC_IN_RC(int_flags));
  4911   match(RegFlags);
  4912   predicate(false);
  4914   format %{ "RFLAGS_U_CF" %}
  4915   interface(REG_INTER);
  4916 %}
  4918 // Float register operands
  4919 operand regF()
  4920 %{
  4921   constraint(ALLOC_IN_RC(float_reg));
  4922   match(RegF);
  4924   format %{ %}
  4925   interface(REG_INTER);
  4926 %}
  4928 // Double register operands
  4929 operand regD() 
  4930 %{
  4931   constraint(ALLOC_IN_RC(double_reg));
  4932   match(RegD);
  4934   format %{ %}
  4935   interface(REG_INTER);
  4936 %}
  4939 //----------Memory Operands----------------------------------------------------
  4940 // Direct Memory Operand
  4941 // operand direct(immP addr)
  4942 // %{
  4943 //   match(addr);
  4945 //   format %{ "[$addr]" %}
  4946 //   interface(MEMORY_INTER) %{
  4947 //     base(0xFFFFFFFF);
  4948 //     index(0x4);
  4949 //     scale(0x0);
  4950 //     disp($addr);
  4951 //   %}
  4952 // %}
  4954 // Indirect Memory Operand
  4955 operand indirect(any_RegP reg)
  4956 %{
  4957   constraint(ALLOC_IN_RC(ptr_reg));
  4958   match(reg);
  4960   format %{ "[$reg]" %}
  4961   interface(MEMORY_INTER) %{
  4962     base($reg);
  4963     index(0x4);
  4964     scale(0x0);
  4965     disp(0x0);
  4966   %}
  4967 %}
  4969 // Indirect Memory Plus Short Offset Operand
  4970 operand indOffset8(any_RegP reg, immL8 off)
  4971 %{
  4972   constraint(ALLOC_IN_RC(ptr_reg));
  4973   match(AddP reg off);
  4975   format %{ "[$reg + $off (8-bit)]" %}
  4976   interface(MEMORY_INTER) %{
  4977     base($reg);
  4978     index(0x4);
  4979     scale(0x0);
  4980     disp($off);
  4981   %}
  4982 %}
  4984 // Indirect Memory Plus Long Offset Operand
  4985 operand indOffset32(any_RegP reg, immL32 off)
  4986 %{
  4987   constraint(ALLOC_IN_RC(ptr_reg));
  4988   match(AddP reg off);
  4990   format %{ "[$reg + $off (32-bit)]" %}
  4991   interface(MEMORY_INTER) %{
  4992     base($reg);
  4993     index(0x4);
  4994     scale(0x0);
  4995     disp($off);
  4996   %}
  4997 %}
  4999 // Indirect Memory Plus Index Register Plus Offset Operand
  5000 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
  5001 %{
  5002   constraint(ALLOC_IN_RC(ptr_reg));
  5003   match(AddP (AddP reg lreg) off);
  5005   op_cost(10);
  5006   format %{"[$reg + $off + $lreg]" %}
  5007   interface(MEMORY_INTER) %{
  5008     base($reg);
  5009     index($lreg);
  5010     scale(0x0);
  5011     disp($off);
  5012   %}
  5013 %}
  5015 // Indirect Memory Plus Index Register Plus Offset Operand
  5016 operand indIndex(any_RegP reg, rRegL lreg)
  5017 %{
  5018   constraint(ALLOC_IN_RC(ptr_reg));
  5019   match(AddP reg lreg);
  5021   op_cost(10);
  5022   format %{"[$reg + $lreg]" %}
  5023   interface(MEMORY_INTER) %{
  5024     base($reg);
  5025     index($lreg);
  5026     scale(0x0);
  5027     disp(0x0);
  5028   %}
  5029 %}
  5031 // Indirect Memory Times Scale Plus Index Register
  5032 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
  5033 %{
  5034   constraint(ALLOC_IN_RC(ptr_reg));
  5035   match(AddP reg (LShiftL lreg scale));
  5037   op_cost(10);
  5038   format %{"[$reg + $lreg << $scale]" %}
  5039   interface(MEMORY_INTER) %{
  5040     base($reg);
  5041     index($lreg);
  5042     scale($scale);
  5043     disp(0x0);
  5044   %}
  5045 %}
  5047 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
  5048 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
  5049 %{
  5050   constraint(ALLOC_IN_RC(ptr_reg));
  5051   match(AddP (AddP reg (LShiftL lreg scale)) off);
  5053   op_cost(10);
  5054   format %{"[$reg + $off + $lreg << $scale]" %}
  5055   interface(MEMORY_INTER) %{
  5056     base($reg);
  5057     index($lreg);
  5058     scale($scale);
  5059     disp($off);
  5060   %}
  5061 %}
  5063 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
  5064 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
  5065 %{
  5066   constraint(ALLOC_IN_RC(ptr_reg));
  5067   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
  5068   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
  5070   op_cost(10);
  5071   format %{"[$reg + $off + $idx << $scale]" %}
  5072   interface(MEMORY_INTER) %{
  5073     base($reg);
  5074     index($idx);
  5075     scale($scale);
  5076     disp($off);
  5077   %}
  5078 %}
  5080 // Indirect Narrow Oop Plus Offset Operand
  5081 // Note: x86 architecture doesn't support "scale * index + offset" without a base
  5082 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
  5083 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
  5084   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
  5085   constraint(ALLOC_IN_RC(ptr_reg));
  5086   match(AddP (DecodeN reg) off);
  5088   op_cost(10);
  5089   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
  5090   interface(MEMORY_INTER) %{
  5091     base(0xc); // R12
  5092     index($reg);
  5093     scale(0x3);
  5094     disp($off);
  5095   %}
  5096 %}
  5098 // Indirect Memory Operand
  5099 operand indirectNarrow(rRegN reg)
  5100 %{
  5101   predicate(Universe::narrow_oop_shift() == 0);
  5102   constraint(ALLOC_IN_RC(ptr_reg));
  5103   match(DecodeN reg);
  5105   format %{ "[$reg]" %}
  5106   interface(MEMORY_INTER) %{
  5107     base($reg);
  5108     index(0x4);
  5109     scale(0x0);
  5110     disp(0x0);
  5111   %}
  5112 %}
  5114 // Indirect Memory Plus Short Offset Operand
  5115 operand indOffset8Narrow(rRegN reg, immL8 off)
  5116 %{
  5117   predicate(Universe::narrow_oop_shift() == 0);
  5118   constraint(ALLOC_IN_RC(ptr_reg));
  5119   match(AddP (DecodeN reg) off);
  5121   format %{ "[$reg + $off (8-bit)]" %}
  5122   interface(MEMORY_INTER) %{
  5123     base($reg);
  5124     index(0x4);
  5125     scale(0x0);
  5126     disp($off);
  5127   %}
  5128 %}
  5130 // Indirect Memory Plus Long Offset Operand
  5131 operand indOffset32Narrow(rRegN reg, immL32 off)
  5132 %{
  5133   predicate(Universe::narrow_oop_shift() == 0);
  5134   constraint(ALLOC_IN_RC(ptr_reg));
  5135   match(AddP (DecodeN reg) off);
  5137   format %{ "[$reg + $off (32-bit)]" %}
  5138   interface(MEMORY_INTER) %{
  5139     base($reg);
  5140     index(0x4);
  5141     scale(0x0);
  5142     disp($off);
  5143   %}
  5144 %}
  5146 // Indirect Memory Plus Index Register Plus Offset Operand
  5147 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
  5148 %{
  5149   predicate(Universe::narrow_oop_shift() == 0);
  5150   constraint(ALLOC_IN_RC(ptr_reg));
  5151   match(AddP (AddP (DecodeN reg) lreg) off);
  5153   op_cost(10);
  5154   format %{"[$reg + $off + $lreg]" %}
  5155   interface(MEMORY_INTER) %{
  5156     base($reg);
  5157     index($lreg);
  5158     scale(0x0);
  5159     disp($off);
  5160   %}
  5161 %}
  5163 // Indirect Memory Plus Index Register Plus Offset Operand
  5164 operand indIndexNarrow(rRegN reg, rRegL lreg)
  5165 %{
  5166   predicate(Universe::narrow_oop_shift() == 0);
  5167   constraint(ALLOC_IN_RC(ptr_reg));
  5168   match(AddP (DecodeN reg) lreg);
  5170   op_cost(10);
  5171   format %{"[$reg + $lreg]" %}
  5172   interface(MEMORY_INTER) %{
  5173     base($reg);
  5174     index($lreg);
  5175     scale(0x0);
  5176     disp(0x0);
  5177   %}
  5178 %}
  5180 // Indirect Memory Times Scale Plus Index Register
  5181 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
  5182 %{
  5183   predicate(Universe::narrow_oop_shift() == 0);
  5184   constraint(ALLOC_IN_RC(ptr_reg));
  5185   match(AddP (DecodeN reg) (LShiftL lreg scale));
  5187   op_cost(10);
  5188   format %{"[$reg + $lreg << $scale]" %}
  5189   interface(MEMORY_INTER) %{
  5190     base($reg);
  5191     index($lreg);
  5192     scale($scale);
  5193     disp(0x0);
  5194   %}
  5195 %}
  5197 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
  5198 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
  5199 %{
  5200   predicate(Universe::narrow_oop_shift() == 0);
  5201   constraint(ALLOC_IN_RC(ptr_reg));
  5202   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
  5204   op_cost(10);
  5205   format %{"[$reg + $off + $lreg << $scale]" %}
  5206   interface(MEMORY_INTER) %{
  5207     base($reg);
  5208     index($lreg);
  5209     scale($scale);
  5210     disp($off);
  5211   %}
  5212 %}
  5214 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
  5215 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
  5216 %{
  5217   constraint(ALLOC_IN_RC(ptr_reg));
  5218   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
  5219   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
  5221   op_cost(10);
  5222   format %{"[$reg + $off + $idx << $scale]" %}
  5223   interface(MEMORY_INTER) %{
  5224     base($reg);
  5225     index($idx);
  5226     scale($scale);
  5227     disp($off);
  5228   %}
  5229 %}
  5232 //----------Special Memory Operands--------------------------------------------
  5233 // Stack Slot Operand - This operand is used for loading and storing temporary
  5234 //                      values on the stack where a match requires a value to
  5235 //                      flow through memory.
  5236 operand stackSlotP(sRegP reg)
  5237 %{
  5238   constraint(ALLOC_IN_RC(stack_slots));
  5239   // No match rule because this operand is only generated in matching
  5241   format %{ "[$reg]" %}
  5242   interface(MEMORY_INTER) %{
  5243     base(0x4);   // RSP
  5244     index(0x4);  // No Index
  5245     scale(0x0);  // No Scale
  5246     disp($reg);  // Stack Offset
  5247   %}
  5248 %}
  5250 operand stackSlotI(sRegI reg)
  5251 %{
  5252   constraint(ALLOC_IN_RC(stack_slots));
  5253   // No match rule because this operand is only generated in matching
  5255   format %{ "[$reg]" %}
  5256   interface(MEMORY_INTER) %{
  5257     base(0x4);   // RSP
  5258     index(0x4);  // No Index
  5259     scale(0x0);  // No Scale
  5260     disp($reg);  // Stack Offset
  5261   %}
  5262 %}
  5264 operand stackSlotF(sRegF reg)
  5265 %{
  5266   constraint(ALLOC_IN_RC(stack_slots));
  5267   // No match rule because this operand is only generated in matching
  5269   format %{ "[$reg]" %}
  5270   interface(MEMORY_INTER) %{
  5271     base(0x4);   // RSP
  5272     index(0x4);  // No Index
  5273     scale(0x0);  // No Scale
  5274     disp($reg);  // Stack Offset
  5275   %}
  5276 %}
  5278 operand stackSlotD(sRegD reg)
  5279 %{
  5280   constraint(ALLOC_IN_RC(stack_slots));
  5281   // No match rule because this operand is only generated in matching
  5283   format %{ "[$reg]" %}
  5284   interface(MEMORY_INTER) %{
  5285     base(0x4);   // RSP
  5286     index(0x4);  // No Index
  5287     scale(0x0);  // No Scale
  5288     disp($reg);  // Stack Offset
  5289   %}
  5290 %}
  5291 operand stackSlotL(sRegL reg)
  5292 %{
  5293   constraint(ALLOC_IN_RC(stack_slots));
  5294   // No match rule because this operand is only generated in matching
  5296   format %{ "[$reg]" %}
  5297   interface(MEMORY_INTER) %{
  5298     base(0x4);   // RSP
  5299     index(0x4);  // No Index
  5300     scale(0x0);  // No Scale
  5301     disp($reg);  // Stack Offset
  5302   %}
  5303 %}
  5305 //----------Conditional Branch Operands----------------------------------------
  5306 // Comparison Op  - This is the operation of the comparison, and is limited to
  5307 //                  the following set of codes:
  5308 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
  5309 //
  5310 // Other attributes of the comparison, such as unsignedness, are specified
  5311 // by the comparison instruction that sets a condition code flags register.
  5312 // That result is represented by a flags operand whose subtype is appropriate
  5313 // to the unsignedness (etc.) of the comparison.
  5314 //
  5315 // Later, the instruction which matches both the Comparison Op (a Bool) and
  5316 // the flags (produced by the Cmp) specifies the coding of the comparison op
  5317 // by matching a specific subtype of Bool operand below, such as cmpOpU.
  5319 // Comparision Code
  5320 operand cmpOp()
  5321 %{
  5322   match(Bool);
  5324   format %{ "" %}
  5325   interface(COND_INTER) %{
  5326     equal(0x4, "e");
  5327     not_equal(0x5, "ne");
  5328     less(0xC, "l");
  5329     greater_equal(0xD, "ge");
  5330     less_equal(0xE, "le");
  5331     greater(0xF, "g");
  5332   %}
  5333 %}
  5335 // Comparison Code, unsigned compare.  Used by FP also, with
  5336 // C2 (unordered) turned into GT or LT already.  The other bits
  5337 // C0 and C3 are turned into Carry & Zero flags.
  5338 operand cmpOpU()
  5339 %{
  5340   match(Bool);
  5342   format %{ "" %}
  5343   interface(COND_INTER) %{
  5344     equal(0x4, "e");
  5345     not_equal(0x5, "ne");
  5346     less(0x2, "b");
  5347     greater_equal(0x3, "nb");
  5348     less_equal(0x6, "be");
  5349     greater(0x7, "nbe");
  5350   %}
  5351 %}
  5354 // Floating comparisons that don't require any fixup for the unordered case
  5355 operand cmpOpUCF() %{
  5356   match(Bool);
  5357   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
  5358             n->as_Bool()->_test._test == BoolTest::ge ||
  5359             n->as_Bool()->_test._test == BoolTest::le ||
  5360             n->as_Bool()->_test._test == BoolTest::gt);
  5361   format %{ "" %}
  5362   interface(COND_INTER) %{
  5363     equal(0x4, "e");
  5364     not_equal(0x5, "ne");
  5365     less(0x2, "b");
  5366     greater_equal(0x3, "nb");
  5367     less_equal(0x6, "be");
  5368     greater(0x7, "nbe");
  5369   %}
  5370 %}
  5373 // Floating comparisons that can be fixed up with extra conditional jumps
  5374 operand cmpOpUCF2() %{
  5375   match(Bool);
  5376   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
  5377             n->as_Bool()->_test._test == BoolTest::eq);
  5378   format %{ "" %}
  5379   interface(COND_INTER) %{
  5380     equal(0x4, "e");
  5381     not_equal(0x5, "ne");
  5382     less(0x2, "b");
  5383     greater_equal(0x3, "nb");
  5384     less_equal(0x6, "be");
  5385     greater(0x7, "nbe");
  5386   %}
  5387 %}
  5390 //----------OPERAND CLASSES----------------------------------------------------
  5391 // Operand Classes are groups of operands that are used as to simplify
  5392 // instruction definitions by not requiring the AD writer to specify separate
  5393 // instructions for every form of operand when the instruction accepts
  5394 // multiple operand types with the same basic encoding and format.  The classic
  5395 // case of this is memory operands.
  5397 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
  5398                indIndexScale, indIndexScaleOffset, indPosIndexScaleOffset,
  5399                indCompressedOopOffset,
  5400                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
  5401                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
  5402                indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow);
  5404 //----------PIPELINE-----------------------------------------------------------
  5405 // Rules which define the behavior of the target architectures pipeline.
  5406 pipeline %{
  5408 //----------ATTRIBUTES---------------------------------------------------------
  5409 attributes %{
  5410   variable_size_instructions;        // Fixed size instructions
  5411   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
  5412   instruction_unit_size = 1;         // An instruction is 1 bytes long
  5413   instruction_fetch_unit_size = 16;  // The processor fetches one line
  5414   instruction_fetch_units = 1;       // of 16 bytes
  5416   // List of nop instructions
  5417   nops( MachNop );
  5418 %}
  5420 //----------RESOURCES----------------------------------------------------------
  5421 // Resources are the functional units available to the machine
  5423 // Generic P2/P3 pipeline
  5424 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
  5425 // 3 instructions decoded per cycle.
  5426 // 2 load/store ops per cycle, 1 branch, 1 FPU,
  5427 // 3 ALU op, only ALU0 handles mul instructions.
  5428 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
  5429            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
  5430            BR, FPU,
  5431            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
  5433 //----------PIPELINE DESCRIPTION-----------------------------------------------
  5434 // Pipeline Description specifies the stages in the machine's pipeline
  5436 // Generic P2/P3 pipeline
  5437 pipe_desc(S0, S1, S2, S3, S4, S5);
  5439 //----------PIPELINE CLASSES---------------------------------------------------
  5440 // Pipeline Classes describe the stages in which input and output are
  5441 // referenced by the hardware pipeline.
  5443 // Naming convention: ialu or fpu
  5444 // Then: _reg
  5445 // Then: _reg if there is a 2nd register
  5446 // Then: _long if it's a pair of instructions implementing a long
  5447 // Then: _fat if it requires the big decoder
  5448 //   Or: _mem if it requires the big decoder and a memory unit.
  5450 // Integer ALU reg operation
  5451 pipe_class ialu_reg(rRegI dst)
  5452 %{
  5453     single_instruction;
  5454     dst    : S4(write);
  5455     dst    : S3(read);
  5456     DECODE : S0;        // any decoder
  5457     ALU    : S3;        // any alu
  5458 %}
  5460 // Long ALU reg operation
  5461 pipe_class ialu_reg_long(rRegL dst)
  5462 %{
  5463     instruction_count(2);
  5464     dst    : S4(write);
  5465     dst    : S3(read);
  5466     DECODE : S0(2);     // any 2 decoders
  5467     ALU    : S3(2);     // both alus
  5468 %}
  5470 // Integer ALU reg operation using big decoder
  5471 pipe_class ialu_reg_fat(rRegI dst)
  5472 %{
  5473     single_instruction;
  5474     dst    : S4(write);
  5475     dst    : S3(read);
  5476     D0     : S0;        // big decoder only
  5477     ALU    : S3;        // any alu
  5478 %}
  5480 // Long ALU reg operation using big decoder
  5481 pipe_class ialu_reg_long_fat(rRegL dst)
  5482 %{
  5483     instruction_count(2);
  5484     dst    : S4(write);
  5485     dst    : S3(read);
  5486     D0     : S0(2);     // big decoder only; twice
  5487     ALU    : S3(2);     // any 2 alus
  5488 %}
  5490 // Integer ALU reg-reg operation
  5491 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
  5492 %{
  5493     single_instruction;
  5494     dst    : S4(write);
  5495     src    : S3(read);
  5496     DECODE : S0;        // any decoder
  5497     ALU    : S3;        // any alu
  5498 %}
  5500 // Long ALU reg-reg operation
  5501 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
  5502 %{
  5503     instruction_count(2);
  5504     dst    : S4(write);
  5505     src    : S3(read);
  5506     DECODE : S0(2);     // any 2 decoders
  5507     ALU    : S3(2);     // both alus
  5508 %}
  5510 // Integer ALU reg-reg operation
  5511 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
  5512 %{
  5513     single_instruction;
  5514     dst    : S4(write);
  5515     src    : S3(read);
  5516     D0     : S0;        // big decoder only
  5517     ALU    : S3;        // any alu
  5518 %}
  5520 // Long ALU reg-reg operation
  5521 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
  5522 %{
  5523     instruction_count(2);
  5524     dst    : S4(write);
  5525     src    : S3(read);
  5526     D0     : S0(2);     // big decoder only; twice
  5527     ALU    : S3(2);     // both alus
  5528 %}
  5530 // Integer ALU reg-mem operation
  5531 pipe_class ialu_reg_mem(rRegI dst, memory mem)
  5532 %{
  5533     single_instruction;
  5534     dst    : S5(write);
  5535     mem    : S3(read);
  5536     D0     : S0;        // big decoder only
  5537     ALU    : S4;        // any alu
  5538     MEM    : S3;        // any mem
  5539 %}
  5541 // Integer mem operation (prefetch)
  5542 pipe_class ialu_mem(memory mem)
  5543 %{
  5544     single_instruction;
  5545     mem    : S3(read);
  5546     D0     : S0;        // big decoder only
  5547     MEM    : S3;        // any mem
  5548 %}
  5550 // Integer Store to Memory
  5551 pipe_class ialu_mem_reg(memory mem, rRegI src)
  5552 %{
  5553     single_instruction;
  5554     mem    : S3(read);
  5555     src    : S5(read);
  5556     D0     : S0;        // big decoder only
  5557     ALU    : S4;        // any alu
  5558     MEM    : S3;
  5559 %}
  5561 // // Long Store to Memory
  5562 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
  5563 // %{
  5564 //     instruction_count(2);
  5565 //     mem    : S3(read);
  5566 //     src    : S5(read);
  5567 //     D0     : S0(2);          // big decoder only; twice
  5568 //     ALU    : S4(2);     // any 2 alus
  5569 //     MEM    : S3(2);  // Both mems
  5570 // %}
  5572 // Integer Store to Memory
  5573 pipe_class ialu_mem_imm(memory mem)
  5574 %{
  5575     single_instruction;
  5576     mem    : S3(read);
  5577     D0     : S0;        // big decoder only
  5578     ALU    : S4;        // any alu
  5579     MEM    : S3;
  5580 %}
  5582 // Integer ALU0 reg-reg operation
  5583 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
  5584 %{
  5585     single_instruction;
  5586     dst    : S4(write);
  5587     src    : S3(read);
  5588     D0     : S0;        // Big decoder only
  5589     ALU0   : S3;        // only alu0
  5590 %}
  5592 // Integer ALU0 reg-mem operation
  5593 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
  5594 %{
  5595     single_instruction;
  5596     dst    : S5(write);
  5597     mem    : S3(read);
  5598     D0     : S0;        // big decoder only
  5599     ALU0   : S4;        // ALU0 only
  5600     MEM    : S3;        // any mem
  5601 %}
  5603 // Integer ALU reg-reg operation
  5604 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
  5605 %{
  5606     single_instruction;
  5607     cr     : S4(write);
  5608     src1   : S3(read);
  5609     src2   : S3(read);
  5610     DECODE : S0;        // any decoder
  5611     ALU    : S3;        // any alu
  5612 %}
  5614 // Integer ALU reg-imm operation
  5615 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
  5616 %{
  5617     single_instruction;
  5618     cr     : S4(write);
  5619     src1   : S3(read);
  5620     DECODE : S0;        // any decoder
  5621     ALU    : S3;        // any alu
  5622 %}
  5624 // Integer ALU reg-mem operation
  5625 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
  5626 %{
  5627     single_instruction;
  5628     cr     : S4(write);
  5629     src1   : S3(read);
  5630     src2   : S3(read);
  5631     D0     : S0;        // big decoder only
  5632     ALU    : S4;        // any alu
  5633     MEM    : S3;
  5634 %}
  5636 // Conditional move reg-reg
  5637 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
  5638 %{
  5639     instruction_count(4);
  5640     y      : S4(read);
  5641     q      : S3(read);
  5642     p      : S3(read);
  5643     DECODE : S0(4);     // any decoder
  5644 %}
  5646 // Conditional move reg-reg
  5647 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
  5648 %{
  5649     single_instruction;
  5650     dst    : S4(write);
  5651     src    : S3(read);
  5652     cr     : S3(read);
  5653     DECODE : S0;        // any decoder
  5654 %}
  5656 // Conditional move reg-mem
  5657 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
  5658 %{
  5659     single_instruction;
  5660     dst    : S4(write);
  5661     src    : S3(read);
  5662     cr     : S3(read);
  5663     DECODE : S0;        // any decoder
  5664     MEM    : S3;
  5665 %}
  5667 // Conditional move reg-reg long
  5668 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
  5669 %{
  5670     single_instruction;
  5671     dst    : S4(write);
  5672     src    : S3(read);
  5673     cr     : S3(read);
  5674     DECODE : S0(2);     // any 2 decoders
  5675 %}
  5677 // XXX
  5678 // // Conditional move double reg-reg
  5679 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
  5680 // %{
  5681 //     single_instruction;
  5682 //     dst    : S4(write);
  5683 //     src    : S3(read);
  5684 //     cr     : S3(read);
  5685 //     DECODE : S0;     // any decoder
  5686 // %}
  5688 // Float reg-reg operation
  5689 pipe_class fpu_reg(regD dst)
  5690 %{
  5691     instruction_count(2);
  5692     dst    : S3(read);
  5693     DECODE : S0(2);     // any 2 decoders
  5694     FPU    : S3;
  5695 %}
  5697 // Float reg-reg operation
  5698 pipe_class fpu_reg_reg(regD dst, regD src)
  5699 %{
  5700     instruction_count(2);
  5701     dst    : S4(write);
  5702     src    : S3(read);
  5703     DECODE : S0(2);     // any 2 decoders
  5704     FPU    : S3;
  5705 %}
  5707 // Float reg-reg operation
  5708 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
  5709 %{
  5710     instruction_count(3);
  5711     dst    : S4(write);
  5712     src1   : S3(read);
  5713     src2   : S3(read);
  5714     DECODE : S0(3);     // any 3 decoders
  5715     FPU    : S3(2);
  5716 %}
  5718 // Float reg-reg operation
  5719 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
  5720 %{
  5721     instruction_count(4);
  5722     dst    : S4(write);
  5723     src1   : S3(read);
  5724     src2   : S3(read);
  5725     src3   : S3(read);
  5726     DECODE : S0(4);     // any 3 decoders
  5727     FPU    : S3(2);
  5728 %}
  5730 // Float reg-reg operation
  5731 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
  5732 %{
  5733     instruction_count(4);
  5734     dst    : S4(write);
  5735     src1   : S3(read);
  5736     src2   : S3(read);
  5737     src3   : S3(read);
  5738     DECODE : S1(3);     // any 3 decoders
  5739     D0     : S0;        // Big decoder only
  5740     FPU    : S3(2);
  5741     MEM    : S3;
  5742 %}
  5744 // Float reg-mem operation
  5745 pipe_class fpu_reg_mem(regD dst, memory mem)
  5746 %{
  5747     instruction_count(2);
  5748     dst    : S5(write);
  5749     mem    : S3(read);
  5750     D0     : S0;        // big decoder only
  5751     DECODE : S1;        // any decoder for FPU POP
  5752     FPU    : S4;
  5753     MEM    : S3;        // any mem
  5754 %}
  5756 // Float reg-mem operation
  5757 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
  5758 %{
  5759     instruction_count(3);
  5760     dst    : S5(write);
  5761     src1   : S3(read);
  5762     mem    : S3(read);
  5763     D0     : S0;        // big decoder only
  5764     DECODE : S1(2);     // any decoder for FPU POP
  5765     FPU    : S4;
  5766     MEM    : S3;        // any mem
  5767 %}
  5769 // Float mem-reg operation
  5770 pipe_class fpu_mem_reg(memory mem, regD src)
  5771 %{
  5772     instruction_count(2);
  5773     src    : S5(read);
  5774     mem    : S3(read);
  5775     DECODE : S0;        // any decoder for FPU PUSH
  5776     D0     : S1;        // big decoder only
  5777     FPU    : S4;
  5778     MEM    : S3;        // any mem
  5779 %}
  5781 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
  5782 %{
  5783     instruction_count(3);
  5784     src1   : S3(read);
  5785     src2   : S3(read);
  5786     mem    : S3(read);
  5787     DECODE : S0(2);     // any decoder for FPU PUSH
  5788     D0     : S1;        // big decoder only
  5789     FPU    : S4;
  5790     MEM    : S3;        // any mem
  5791 %}
  5793 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
  5794 %{
  5795     instruction_count(3);
  5796     src1   : S3(read);
  5797     src2   : S3(read);
  5798     mem    : S4(read);
  5799     DECODE : S0;        // any decoder for FPU PUSH
  5800     D0     : S0(2);     // big decoder only
  5801     FPU    : S4;
  5802     MEM    : S3(2);     // any mem
  5803 %}
  5805 pipe_class fpu_mem_mem(memory dst, memory src1)
  5806 %{
  5807     instruction_count(2);
  5808     src1   : S3(read);
  5809     dst    : S4(read);
  5810     D0     : S0(2);     // big decoder only
  5811     MEM    : S3(2);     // any mem
  5812 %}
  5814 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
  5815 %{
  5816     instruction_count(3);
  5817     src1   : S3(read);
  5818     src2   : S3(read);
  5819     dst    : S4(read);
  5820     D0     : S0(3);     // big decoder only
  5821     FPU    : S4;
  5822     MEM    : S3(3);     // any mem
  5823 %}
  5825 pipe_class fpu_mem_reg_con(memory mem, regD src1)
  5826 %{
  5827     instruction_count(3);
  5828     src1   : S4(read);
  5829     mem    : S4(read);
  5830     DECODE : S0;        // any decoder for FPU PUSH
  5831     D0     : S0(2);     // big decoder only
  5832     FPU    : S4;
  5833     MEM    : S3(2);     // any mem
  5834 %}
  5836 // Float load constant
  5837 pipe_class fpu_reg_con(regD dst)
  5838 %{
  5839     instruction_count(2);
  5840     dst    : S5(write);
  5841     D0     : S0;        // big decoder only for the load
  5842     DECODE : S1;        // any decoder for FPU POP
  5843     FPU    : S4;
  5844     MEM    : S3;        // any mem
  5845 %}
  5847 // Float load constant
  5848 pipe_class fpu_reg_reg_con(regD dst, regD src)
  5849 %{
  5850     instruction_count(3);
  5851     dst    : S5(write);
  5852     src    : S3(read);
  5853     D0     : S0;        // big decoder only for the load
  5854     DECODE : S1(2);     // any decoder for FPU POP
  5855     FPU    : S4;
  5856     MEM    : S3;        // any mem
  5857 %}
  5859 // UnConditional branch
  5860 pipe_class pipe_jmp(label labl)
  5861 %{
  5862     single_instruction;
  5863     BR   : S3;
  5864 %}
  5866 // Conditional branch
  5867 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
  5868 %{
  5869     single_instruction;
  5870     cr    : S1(read);
  5871     BR    : S3;
  5872 %}
  5874 // Allocation idiom
  5875 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
  5876 %{
  5877     instruction_count(1); force_serialization;
  5878     fixed_latency(6);
  5879     heap_ptr : S3(read);
  5880     DECODE   : S0(3);
  5881     D0       : S2;
  5882     MEM      : S3;
  5883     ALU      : S3(2);
  5884     dst      : S5(write);
  5885     BR       : S5;
  5886 %}
  5888 // Generic big/slow expanded idiom
  5889 pipe_class pipe_slow()
  5890 %{
  5891     instruction_count(10); multiple_bundles; force_serialization;
  5892     fixed_latency(100);
  5893     D0  : S0(2);
  5894     MEM : S3(2);
  5895 %}
  5897 // The real do-nothing guy
  5898 pipe_class empty()
  5899 %{
  5900     instruction_count(0);
  5901 %}
  5903 // Define the class for the Nop node
  5904 define
  5905 %{
  5906    MachNop = empty;
  5907 %}
  5909 %}
  5911 //----------INSTRUCTIONS-------------------------------------------------------
  5912 //
  5913 // match      -- States which machine-independent subtree may be replaced
  5914 //               by this instruction.
  5915 // ins_cost   -- The estimated cost of this instruction is used by instruction
  5916 //               selection to identify a minimum cost tree of machine
  5917 //               instructions that matches a tree of machine-independent
  5918 //               instructions.
  5919 // format     -- A string providing the disassembly for this instruction.
  5920 //               The value of an instruction's operand may be inserted
  5921 //               by referring to it with a '$' prefix.
  5922 // opcode     -- Three instruction opcodes may be provided.  These are referred
  5923 //               to within an encode class as $primary, $secondary, and $tertiary
  5924 //               rrspectively.  The primary opcode is commonly used to
  5925 //               indicate the type of machine instruction, while secondary
  5926 //               and tertiary are often used for prefix options or addressing
  5927 //               modes.
  5928 // ins_encode -- A list of encode classes with parameters. The encode class
  5929 //               name must have been defined in an 'enc_class' specification
  5930 //               in the encode section of the architecture description.
  5933 //----------Load/Store/Move Instructions---------------------------------------
  5934 //----------Load Instructions--------------------------------------------------
  5936 // Load Byte (8 bit signed)
  5937 instruct loadB(rRegI dst, memory mem)
  5938 %{
  5939   match(Set dst (LoadB mem));
  5941   ins_cost(125);
  5942   format %{ "movsbl  $dst, $mem\t# byte" %}
  5944   ins_encode %{
  5945     __ movsbl($dst$$Register, $mem$$Address);
  5946   %}
  5948   ins_pipe(ialu_reg_mem);
  5949 %}
  5951 // Load Byte (8 bit signed) into Long Register
  5952 instruct loadB2L(rRegL dst, memory mem)
  5953 %{
  5954   match(Set dst (ConvI2L (LoadB mem)));
  5956   ins_cost(125);
  5957   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
  5959   ins_encode %{
  5960     __ movsbq($dst$$Register, $mem$$Address);
  5961   %}
  5963   ins_pipe(ialu_reg_mem);
  5964 %}
  5966 // Load Unsigned Byte (8 bit UNsigned)
  5967 instruct loadUB(rRegI dst, memory mem)
  5968 %{
  5969   match(Set dst (LoadUB mem));
  5971   ins_cost(125);
  5972   format %{ "movzbl  $dst, $mem\t# ubyte" %}
  5974   ins_encode %{
  5975     __ movzbl($dst$$Register, $mem$$Address);
  5976   %}
  5978   ins_pipe(ialu_reg_mem);
  5979 %}
  5981 // Load Unsigned Byte (8 bit UNsigned) into Long Register
  5982 instruct loadUB2L(rRegL dst, memory mem)
  5983 %{
  5984   match(Set dst (ConvI2L (LoadUB mem)));
  5986   ins_cost(125);
  5987   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
  5989   ins_encode %{
  5990     __ movzbq($dst$$Register, $mem$$Address);
  5991   %}
  5993   ins_pipe(ialu_reg_mem);
  5994 %}
  5996 // Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
  5997 instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
  5998   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
  5999   effect(KILL cr);
  6001   format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
  6002             "andl    $dst, $mask" %}
  6003   ins_encode %{
  6004     Register Rdst = $dst$$Register;
  6005     __ movzbq(Rdst, $mem$$Address);
  6006     __ andl(Rdst, $mask$$constant);
  6007   %}
  6008   ins_pipe(ialu_reg_mem);
  6009 %}
  6011 // Load Short (16 bit signed)
  6012 instruct loadS(rRegI dst, memory mem)
  6013 %{
  6014   match(Set dst (LoadS mem));
  6016   ins_cost(125);
  6017   format %{ "movswl $dst, $mem\t# short" %}
  6019   ins_encode %{
  6020     __ movswl($dst$$Register, $mem$$Address);
  6021   %}
  6023   ins_pipe(ialu_reg_mem);
  6024 %}
  6026 // Load Short (16 bit signed) to Byte (8 bit signed)
  6027 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
  6028   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
  6030   ins_cost(125);
  6031   format %{ "movsbl $dst, $mem\t# short -> byte" %}
  6032   ins_encode %{
  6033     __ movsbl($dst$$Register, $mem$$Address);
  6034   %}
  6035   ins_pipe(ialu_reg_mem);
  6036 %}
  6038 // Load Short (16 bit signed) into Long Register
  6039 instruct loadS2L(rRegL dst, memory mem)
  6040 %{
  6041   match(Set dst (ConvI2L (LoadS mem)));
  6043   ins_cost(125);
  6044   format %{ "movswq $dst, $mem\t# short -> long" %}
  6046   ins_encode %{
  6047     __ movswq($dst$$Register, $mem$$Address);
  6048   %}
  6050   ins_pipe(ialu_reg_mem);
  6051 %}
  6053 // Load Unsigned Short/Char (16 bit UNsigned)
  6054 instruct loadUS(rRegI dst, memory mem)
  6055 %{
  6056   match(Set dst (LoadUS mem));
  6058   ins_cost(125);
  6059   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
  6061   ins_encode %{
  6062     __ movzwl($dst$$Register, $mem$$Address);
  6063   %}
  6065   ins_pipe(ialu_reg_mem);
  6066 %}
  6068 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
  6069 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
  6070   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
  6072   ins_cost(125);
  6073   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
  6074   ins_encode %{
  6075     __ movsbl($dst$$Register, $mem$$Address);
  6076   %}
  6077   ins_pipe(ialu_reg_mem);
  6078 %}
  6080 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
  6081 instruct loadUS2L(rRegL dst, memory mem)
  6082 %{
  6083   match(Set dst (ConvI2L (LoadUS mem)));
  6085   ins_cost(125);
  6086   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
  6088   ins_encode %{
  6089     __ movzwq($dst$$Register, $mem$$Address);
  6090   %}
  6092   ins_pipe(ialu_reg_mem);
  6093 %}
  6095 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
  6096 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
  6097   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
  6099   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
  6100   ins_encode %{
  6101     __ movzbq($dst$$Register, $mem$$Address);
  6102   %}
  6103   ins_pipe(ialu_reg_mem);
  6104 %}
  6106 // Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
  6107 instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
  6108   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
  6109   effect(KILL cr);
  6111   format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
  6112             "andl    $dst, $mask" %}
  6113   ins_encode %{
  6114     Register Rdst = $dst$$Register;
  6115     __ movzwq(Rdst, $mem$$Address);
  6116     __ andl(Rdst, $mask$$constant);
  6117   %}
  6118   ins_pipe(ialu_reg_mem);
  6119 %}
  6121 // Load Integer
  6122 instruct loadI(rRegI dst, memory mem)
  6123 %{
  6124   match(Set dst (LoadI mem));
  6126   ins_cost(125);
  6127   format %{ "movl    $dst, $mem\t# int" %}
  6129   ins_encode %{
  6130     __ movl($dst$$Register, $mem$$Address);
  6131   %}
  6133   ins_pipe(ialu_reg_mem);
  6134 %}
  6136 // Load Integer (32 bit signed) to Byte (8 bit signed)
  6137 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
  6138   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
  6140   ins_cost(125);
  6141   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
  6142   ins_encode %{
  6143     __ movsbl($dst$$Register, $mem$$Address);
  6144   %}
  6145   ins_pipe(ialu_reg_mem);
  6146 %}
  6148 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
  6149 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
  6150   match(Set dst (AndI (LoadI mem) mask));
  6152   ins_cost(125);
  6153   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
  6154   ins_encode %{
  6155     __ movzbl($dst$$Register, $mem$$Address);
  6156   %}
  6157   ins_pipe(ialu_reg_mem);
  6158 %}
  6160 // Load Integer (32 bit signed) to Short (16 bit signed)
  6161 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
  6162   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
  6164   ins_cost(125);
  6165   format %{ "movswl  $dst, $mem\t# int -> short" %}
  6166   ins_encode %{
  6167     __ movswl($dst$$Register, $mem$$Address);
  6168   %}
  6169   ins_pipe(ialu_reg_mem);
  6170 %}
  6172 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
  6173 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
  6174   match(Set dst (AndI (LoadI mem) mask));
  6176   ins_cost(125);
  6177   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
  6178   ins_encode %{
  6179     __ movzwl($dst$$Register, $mem$$Address);
  6180   %}
  6181   ins_pipe(ialu_reg_mem);
  6182 %}
  6184 // Load Integer into Long Register
  6185 instruct loadI2L(rRegL dst, memory mem)
  6186 %{
  6187   match(Set dst (ConvI2L (LoadI mem)));
  6189   ins_cost(125);
  6190   format %{ "movslq  $dst, $mem\t# int -> long" %}
  6192   ins_encode %{
  6193     __ movslq($dst$$Register, $mem$$Address);
  6194   %}
  6196   ins_pipe(ialu_reg_mem);
  6197 %}
  6199 // Load Integer with mask 0xFF into Long Register
  6200 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
  6201   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  6203   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
  6204   ins_encode %{
  6205     __ movzbq($dst$$Register, $mem$$Address);
  6206   %}
  6207   ins_pipe(ialu_reg_mem);
  6208 %}
  6210 // Load Integer with mask 0xFFFF into Long Register
  6211 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
  6212   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  6214   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
  6215   ins_encode %{
  6216     __ movzwq($dst$$Register, $mem$$Address);
  6217   %}
  6218   ins_pipe(ialu_reg_mem);
  6219 %}
  6221 // Load Integer with a 32-bit mask into Long Register
  6222 instruct loadI2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
  6223   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  6224   effect(KILL cr);
  6226   format %{ "movl    $dst, $mem\t# int & 32-bit mask -> long\n\t"
  6227             "andl    $dst, $mask" %}
  6228   ins_encode %{
  6229     Register Rdst = $dst$$Register;
  6230     __ movl(Rdst, $mem$$Address);
  6231     __ andl(Rdst, $mask$$constant);
  6232   %}
  6233   ins_pipe(ialu_reg_mem);
  6234 %}
  6236 // Load Unsigned Integer into Long Register
  6237 instruct loadUI2L(rRegL dst, memory mem)
  6238 %{
  6239   match(Set dst (LoadUI2L mem));
  6241   ins_cost(125);
  6242   format %{ "movl    $dst, $mem\t# uint -> long" %}
  6244   ins_encode %{
  6245     __ movl($dst$$Register, $mem$$Address);
  6246   %}
  6248   ins_pipe(ialu_reg_mem);
  6249 %}
  6251 // Load Long
  6252 instruct loadL(rRegL dst, memory mem)
  6253 %{
  6254   match(Set dst (LoadL mem));
  6256   ins_cost(125);
  6257   format %{ "movq    $dst, $mem\t# long" %}
  6259   ins_encode %{
  6260     __ movq($dst$$Register, $mem$$Address);
  6261   %}
  6263   ins_pipe(ialu_reg_mem); // XXX
  6264 %}
  6266 // Load Range
  6267 instruct loadRange(rRegI dst, memory mem)
  6268 %{
  6269   match(Set dst (LoadRange mem));
  6271   ins_cost(125); // XXX
  6272   format %{ "movl    $dst, $mem\t# range" %}
  6273   opcode(0x8B);
  6274   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
  6275   ins_pipe(ialu_reg_mem);
  6276 %}
  6278 // Load Pointer
  6279 instruct loadP(rRegP dst, memory mem)
  6280 %{
  6281   match(Set dst (LoadP mem));
  6283   ins_cost(125); // XXX
  6284   format %{ "movq    $dst, $mem\t# ptr" %}
  6285   opcode(0x8B);
  6286   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6287   ins_pipe(ialu_reg_mem); // XXX
  6288 %}
  6290 // Load Compressed Pointer
  6291 instruct loadN(rRegN dst, memory mem)
  6292 %{
  6293    match(Set dst (LoadN mem));
  6295    ins_cost(125); // XXX
  6296    format %{ "movl    $dst, $mem\t# compressed ptr" %}
  6297    ins_encode %{
  6298      __ movl($dst$$Register, $mem$$Address);
  6299    %}
  6300    ins_pipe(ialu_reg_mem); // XXX
  6301 %}
  6304 // Load Klass Pointer
  6305 instruct loadKlass(rRegP dst, memory mem)
  6306 %{
  6307   match(Set dst (LoadKlass mem));
  6309   ins_cost(125); // XXX
  6310   format %{ "movq    $dst, $mem\t# class" %}
  6311   opcode(0x8B);
  6312   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6313   ins_pipe(ialu_reg_mem); // XXX
  6314 %}
  6316 // Load narrow Klass Pointer
  6317 instruct loadNKlass(rRegN dst, memory mem)
  6318 %{
  6319   match(Set dst (LoadNKlass mem));
  6321   ins_cost(125); // XXX
  6322   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
  6323   ins_encode %{
  6324     __ movl($dst$$Register, $mem$$Address);
  6325   %}
  6326   ins_pipe(ialu_reg_mem); // XXX
  6327 %}
  6329 // Load Float
  6330 instruct loadF(regF dst, memory mem)
  6331 %{
  6332   match(Set dst (LoadF mem));
  6334   ins_cost(145); // XXX
  6335   format %{ "movss   $dst, $mem\t# float" %}
  6336   opcode(0xF3, 0x0F, 0x10);
  6337   ins_encode(OpcP, REX_reg_mem(dst, mem), OpcS, OpcT, reg_mem(dst, mem));
  6338   ins_pipe(pipe_slow); // XXX
  6339 %}
  6341 // Load Double
  6342 instruct loadD_partial(regD dst, memory mem)
  6343 %{
  6344   predicate(!UseXmmLoadAndClearUpper);
  6345   match(Set dst (LoadD mem));
  6347   ins_cost(145); // XXX
  6348   format %{ "movlpd  $dst, $mem\t# double" %}
  6349   opcode(0x66, 0x0F, 0x12);
  6350   ins_encode(OpcP, REX_reg_mem(dst, mem), OpcS, OpcT, reg_mem(dst, mem));
  6351   ins_pipe(pipe_slow); // XXX
  6352 %}
  6354 instruct loadD(regD dst, memory mem)
  6355 %{
  6356   predicate(UseXmmLoadAndClearUpper);
  6357   match(Set dst (LoadD mem));
  6359   ins_cost(145); // XXX
  6360   format %{ "movsd   $dst, $mem\t# double" %}
  6361   opcode(0xF2, 0x0F, 0x10);
  6362   ins_encode(OpcP, REX_reg_mem(dst, mem), OpcS, OpcT, reg_mem(dst, mem));
  6363   ins_pipe(pipe_slow); // XXX
  6364 %}
  6366 // Load Aligned Packed Byte to XMM register
  6367 instruct loadA8B(regD dst, memory mem) %{
  6368   match(Set dst (Load8B mem));
  6369   ins_cost(125);
  6370   format %{ "MOVQ  $dst,$mem\t! packed8B" %}
  6371   ins_encode( movq_ld(dst, mem));
  6372   ins_pipe( pipe_slow );
  6373 %}
  6375 // Load Aligned Packed Short to XMM register
  6376 instruct loadA4S(regD dst, memory mem) %{
  6377   match(Set dst (Load4S mem));
  6378   ins_cost(125);
  6379   format %{ "MOVQ  $dst,$mem\t! packed4S" %}
  6380   ins_encode( movq_ld(dst, mem));
  6381   ins_pipe( pipe_slow );
  6382 %}
  6384 // Load Aligned Packed Char to XMM register
  6385 instruct loadA4C(regD dst, memory mem) %{
  6386   match(Set dst (Load4C mem));
  6387   ins_cost(125);
  6388   format %{ "MOVQ  $dst,$mem\t! packed4C" %}
  6389   ins_encode( movq_ld(dst, mem));
  6390   ins_pipe( pipe_slow );
  6391 %}
  6393 // Load Aligned Packed Integer to XMM register
  6394 instruct load2IU(regD dst, memory mem) %{
  6395   match(Set dst (Load2I mem));
  6396   ins_cost(125);
  6397   format %{ "MOVQ  $dst,$mem\t! packed2I" %}
  6398   ins_encode( movq_ld(dst, mem));
  6399   ins_pipe( pipe_slow );
  6400 %}
  6402 // Load Aligned Packed Single to XMM
  6403 instruct loadA2F(regD dst, memory mem) %{
  6404   match(Set dst (Load2F mem));
  6405   ins_cost(145);
  6406   format %{ "MOVQ  $dst,$mem\t! packed2F" %}
  6407   ins_encode( movq_ld(dst, mem));
  6408   ins_pipe( pipe_slow );
  6409 %}
  6411 // Load Effective Address
  6412 instruct leaP8(rRegP dst, indOffset8 mem)
  6413 %{
  6414   match(Set dst mem);
  6416   ins_cost(110); // XXX
  6417   format %{ "leaq    $dst, $mem\t# ptr 8" %}
  6418   opcode(0x8D);
  6419   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6420   ins_pipe(ialu_reg_reg_fat);
  6421 %}
  6423 instruct leaP32(rRegP dst, indOffset32 mem)
  6424 %{
  6425   match(Set dst mem);
  6427   ins_cost(110);
  6428   format %{ "leaq    $dst, $mem\t# ptr 32" %}
  6429   opcode(0x8D);
  6430   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6431   ins_pipe(ialu_reg_reg_fat);
  6432 %}
  6434 // instruct leaPIdx(rRegP dst, indIndex mem)
  6435 // %{
  6436 //   match(Set dst mem);
  6438 //   ins_cost(110);
  6439 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
  6440 //   opcode(0x8D);
  6441 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6442 //   ins_pipe(ialu_reg_reg_fat);
  6443 // %}
  6445 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
  6446 %{
  6447   match(Set dst mem);
  6449   ins_cost(110);
  6450   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
  6451   opcode(0x8D);
  6452   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6453   ins_pipe(ialu_reg_reg_fat);
  6454 %}
  6456 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
  6457 %{
  6458   match(Set dst mem);
  6460   ins_cost(110);
  6461   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
  6462   opcode(0x8D);
  6463   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6464   ins_pipe(ialu_reg_reg_fat);
  6465 %}
  6467 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
  6468 %{
  6469   match(Set dst mem);
  6471   ins_cost(110);
  6472   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
  6473   opcode(0x8D);
  6474   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6475   ins_pipe(ialu_reg_reg_fat);
  6476 %}
  6478 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
  6479 %{
  6480   match(Set dst mem);
  6482   ins_cost(110);
  6483   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
  6484   opcode(0x8D);
  6485   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6486   ins_pipe(ialu_reg_reg_fat);
  6487 %}
  6489 // Load Effective Address which uses Narrow (32-bits) oop
  6490 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
  6491 %{
  6492   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
  6493   match(Set dst mem);
  6495   ins_cost(110);
  6496   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
  6497   opcode(0x8D);
  6498   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6499   ins_pipe(ialu_reg_reg_fat);
  6500 %}
  6502 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
  6503 %{
  6504   predicate(Universe::narrow_oop_shift() == 0);
  6505   match(Set dst mem);
  6507   ins_cost(110); // XXX
  6508   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
  6509   opcode(0x8D);
  6510   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6511   ins_pipe(ialu_reg_reg_fat);
  6512 %}
  6514 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
  6515 %{
  6516   predicate(Universe::narrow_oop_shift() == 0);
  6517   match(Set dst mem);
  6519   ins_cost(110);
  6520   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
  6521   opcode(0x8D);
  6522   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6523   ins_pipe(ialu_reg_reg_fat);
  6524 %}
  6526 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
  6527 %{
  6528   predicate(Universe::narrow_oop_shift() == 0);
  6529   match(Set dst mem);
  6531   ins_cost(110);
  6532   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
  6533   opcode(0x8D);
  6534   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6535   ins_pipe(ialu_reg_reg_fat);
  6536 %}
  6538 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
  6539 %{
  6540   predicate(Universe::narrow_oop_shift() == 0);
  6541   match(Set dst mem);
  6543   ins_cost(110);
  6544   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
  6545   opcode(0x8D);
  6546   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6547   ins_pipe(ialu_reg_reg_fat);
  6548 %}
  6550 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
  6551 %{
  6552   predicate(Universe::narrow_oop_shift() == 0);
  6553   match(Set dst mem);
  6555   ins_cost(110);
  6556   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
  6557   opcode(0x8D);
  6558   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6559   ins_pipe(ialu_reg_reg_fat);
  6560 %}
  6562 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
  6563 %{
  6564   predicate(Universe::narrow_oop_shift() == 0);
  6565   match(Set dst mem);
  6567   ins_cost(110);
  6568   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
  6569   opcode(0x8D);
  6570   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  6571   ins_pipe(ialu_reg_reg_fat);
  6572 %}
  6574 instruct loadConI(rRegI dst, immI src)
  6575 %{
  6576   match(Set dst src);
  6578   format %{ "movl    $dst, $src\t# int" %}
  6579   ins_encode(load_immI(dst, src));
  6580   ins_pipe(ialu_reg_fat); // XXX
  6581 %}
  6583 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
  6584 %{
  6585   match(Set dst src);
  6586   effect(KILL cr);
  6588   ins_cost(50);
  6589   format %{ "xorl    $dst, $dst\t# int" %}
  6590   opcode(0x33); /* + rd */
  6591   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
  6592   ins_pipe(ialu_reg);
  6593 %}
  6595 instruct loadConL(rRegL dst, immL src)
  6596 %{
  6597   match(Set dst src);
  6599   ins_cost(150);
  6600   format %{ "movq    $dst, $src\t# long" %}
  6601   ins_encode(load_immL(dst, src));
  6602   ins_pipe(ialu_reg);
  6603 %}
  6605 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
  6606 %{
  6607   match(Set dst src);
  6608   effect(KILL cr);
  6610   ins_cost(50);
  6611   format %{ "xorl    $dst, $dst\t# long" %}
  6612   opcode(0x33); /* + rd */
  6613   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
  6614   ins_pipe(ialu_reg); // XXX
  6615 %}
  6617 instruct loadConUL32(rRegL dst, immUL32 src)
  6618 %{
  6619   match(Set dst src);
  6621   ins_cost(60);
  6622   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
  6623   ins_encode(load_immUL32(dst, src));
  6624   ins_pipe(ialu_reg);
  6625 %}
  6627 instruct loadConL32(rRegL dst, immL32 src)
  6628 %{
  6629   match(Set dst src);
  6631   ins_cost(70);
  6632   format %{ "movq    $dst, $src\t# long (32-bit)" %}
  6633   ins_encode(load_immL32(dst, src));
  6634   ins_pipe(ialu_reg);
  6635 %}
  6637 instruct loadConP(rRegP dst, immP src)
  6638 %{
  6639   match(Set dst src);
  6641   format %{ "movq    $dst, $src\t# ptr" %}
  6642   ins_encode(load_immP(dst, src));
  6643   ins_pipe(ialu_reg_fat); // XXX
  6644 %}
  6646 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
  6647 %{
  6648   match(Set dst src);
  6649   effect(KILL cr);
  6651   ins_cost(50);
  6652   format %{ "xorl    $dst, $dst\t# ptr" %}
  6653   opcode(0x33); /* + rd */
  6654   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
  6655   ins_pipe(ialu_reg);
  6656 %}
  6658 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
  6659 %{
  6660   match(Set dst src);
  6661   effect(KILL cr);
  6663   ins_cost(60);
  6664   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
  6665   ins_encode(load_immP31(dst, src));
  6666   ins_pipe(ialu_reg);
  6667 %}
  6669 instruct loadConF(regF dst, immF src)
  6670 %{
  6671   match(Set dst src);
  6672   ins_cost(125);
  6674   format %{ "movss   $dst, [$src]" %}
  6675   ins_encode(load_conF(dst, src));
  6676   ins_pipe(pipe_slow);
  6677 %}
  6679 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
  6680   match(Set dst src);
  6681   effect(KILL cr);
  6682   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
  6683   ins_encode %{
  6684     __ xorq($dst$$Register, $dst$$Register);
  6685   %}
  6686   ins_pipe(ialu_reg);
  6687 %}
  6689 instruct loadConN(rRegN dst, immN src) %{
  6690   match(Set dst src);
  6692   ins_cost(125);
  6693   format %{ "movl    $dst, $src\t# compressed ptr" %}
  6694   ins_encode %{
  6695     address con = (address)$src$$constant;
  6696     if (con == NULL) {
  6697       ShouldNotReachHere();
  6698     } else {
  6699       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
  6701   %}
  6702   ins_pipe(ialu_reg_fat); // XXX
  6703 %}
  6705 instruct loadConF0(regF dst, immF0 src)
  6706 %{
  6707   match(Set dst src);
  6708   ins_cost(100);
  6710   format %{ "xorps   $dst, $dst\t# float 0.0" %}
  6711   opcode(0x0F, 0x57);
  6712   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
  6713   ins_pipe(pipe_slow);
  6714 %}
  6716 // Use the same format since predicate() can not be used here.
  6717 instruct loadConD(regD dst, immD src)
  6718 %{
  6719   match(Set dst src);
  6720   ins_cost(125);
  6722   format %{ "movsd   $dst, [$src]" %}
  6723   ins_encode(load_conD(dst, src));
  6724   ins_pipe(pipe_slow);
  6725 %}
  6727 instruct loadConD0(regD dst, immD0 src)
  6728 %{
  6729   match(Set dst src);
  6730   ins_cost(100);
  6732   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
  6733   opcode(0x66, 0x0F, 0x57);
  6734   ins_encode(OpcP, REX_reg_reg(dst, dst), OpcS, OpcT, reg_reg(dst, dst));
  6735   ins_pipe(pipe_slow);
  6736 %}
  6738 instruct loadSSI(rRegI dst, stackSlotI src)
  6739 %{
  6740   match(Set dst src);
  6742   ins_cost(125);
  6743   format %{ "movl    $dst, $src\t# int stk" %}
  6744   opcode(0x8B);
  6745   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
  6746   ins_pipe(ialu_reg_mem);
  6747 %}
  6749 instruct loadSSL(rRegL dst, stackSlotL src)
  6750 %{
  6751   match(Set dst src);
  6753   ins_cost(125);
  6754   format %{ "movq    $dst, $src\t# long stk" %}
  6755   opcode(0x8B);
  6756   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
  6757   ins_pipe(ialu_reg_mem);
  6758 %}
  6760 instruct loadSSP(rRegP dst, stackSlotP src)
  6761 %{
  6762   match(Set dst src);
  6764   ins_cost(125);
  6765   format %{ "movq    $dst, $src\t# ptr stk" %}
  6766   opcode(0x8B);
  6767   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
  6768   ins_pipe(ialu_reg_mem);
  6769 %}
  6771 instruct loadSSF(regF dst, stackSlotF src)
  6772 %{
  6773   match(Set dst src);
  6775   ins_cost(125);
  6776   format %{ "movss   $dst, $src\t# float stk" %}
  6777   opcode(0xF3, 0x0F, 0x10);
  6778   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
  6779   ins_pipe(pipe_slow); // XXX
  6780 %}
  6782 // Use the same format since predicate() can not be used here.
  6783 instruct loadSSD(regD dst, stackSlotD src)
  6784 %{
  6785   match(Set dst src);
  6787   ins_cost(125);
  6788   format %{ "movsd   $dst, $src\t# double stk" %}
  6789   ins_encode  %{
  6790     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
  6791   %}
  6792   ins_pipe(pipe_slow); // XXX
  6793 %}
  6795 // Prefetch instructions.
  6796 // Must be safe to execute with invalid address (cannot fault).
  6798 instruct prefetchr( memory mem ) %{
  6799   predicate(ReadPrefetchInstr==3);
  6800   match(PrefetchRead mem);
  6801   ins_cost(125);
  6803   format %{ "PREFETCHR $mem\t# Prefetch into level 1 cache" %}
  6804   opcode(0x0F, 0x0D);     /* Opcode 0F 0D /0 */
  6805   ins_encode(REX_mem(mem), OpcP, OpcS, RM_opc_mem(0x00, mem));
  6806   ins_pipe(ialu_mem);
  6807 %}
  6809 instruct prefetchrNTA( memory mem ) %{
  6810   predicate(ReadPrefetchInstr==0);
  6811   match(PrefetchRead mem);
  6812   ins_cost(125);
  6814   format %{ "PREFETCHNTA $mem\t# Prefetch into non-temporal cache for read" %}
  6815   opcode(0x0F, 0x18);     /* Opcode 0F 18 /0 */
  6816   ins_encode(REX_mem(mem), OpcP, OpcS, RM_opc_mem(0x00, mem));
  6817   ins_pipe(ialu_mem);
  6818 %}
  6820 instruct prefetchrT0( memory mem ) %{
  6821   predicate(ReadPrefetchInstr==1);
  6822   match(PrefetchRead mem);
  6823   ins_cost(125);
  6825   format %{ "PREFETCHT0 $mem\t# prefetch into L1 and L2 caches for read" %}
  6826   opcode(0x0F, 0x18); /* Opcode 0F 18 /1 */
  6827   ins_encode(REX_mem(mem), OpcP, OpcS, RM_opc_mem(0x01, mem));
  6828   ins_pipe(ialu_mem);
  6829 %}
  6831 instruct prefetchrT2( memory mem ) %{
  6832   predicate(ReadPrefetchInstr==2);
  6833   match(PrefetchRead mem);
  6834   ins_cost(125);
  6836   format %{ "PREFETCHT2 $mem\t# prefetch into L2 caches for read" %}
  6837   opcode(0x0F, 0x18); /* Opcode 0F 18 /3 */
  6838   ins_encode(REX_mem(mem), OpcP, OpcS, RM_opc_mem(0x03, mem));
  6839   ins_pipe(ialu_mem);
  6840 %}
  6842 instruct prefetchw( memory mem ) %{
  6843   predicate(AllocatePrefetchInstr==3);
  6844   match(PrefetchWrite mem);
  6845   ins_cost(125);
  6847   format %{ "PREFETCHW $mem\t# Prefetch into level 1 cache and mark modified" %}
  6848   opcode(0x0F, 0x0D);     /* Opcode 0F 0D /1 */
  6849   ins_encode(REX_mem(mem), OpcP, OpcS, RM_opc_mem(0x01, mem));
  6850   ins_pipe(ialu_mem);
  6851 %}
  6853 instruct prefetchwNTA( memory mem ) %{
  6854   predicate(AllocatePrefetchInstr==0);
  6855   match(PrefetchWrite mem);
  6856   ins_cost(125);
  6858   format %{ "PREFETCHNTA $mem\t# Prefetch to non-temporal cache for write" %}
  6859   opcode(0x0F, 0x18);     /* Opcode 0F 18 /0 */
  6860   ins_encode(REX_mem(mem), OpcP, OpcS, RM_opc_mem(0x00, mem));
  6861   ins_pipe(ialu_mem);
  6862 %}
  6864 instruct prefetchwT0( memory mem ) %{
  6865   predicate(AllocatePrefetchInstr==1);
  6866   match(PrefetchWrite mem);
  6867   ins_cost(125);
  6869   format %{ "PREFETCHT0 $mem\t# Prefetch to level 1 and 2 caches for write" %}
  6870   opcode(0x0F, 0x18);     /* Opcode 0F 18 /1 */
  6871   ins_encode(REX_mem(mem), OpcP, OpcS, RM_opc_mem(0x01, mem));
  6872   ins_pipe(ialu_mem);
  6873 %}
  6875 instruct prefetchwT2( memory mem ) %{
  6876   predicate(AllocatePrefetchInstr==2);
  6877   match(PrefetchWrite mem);
  6878   ins_cost(125);
  6880   format %{ "PREFETCHT2 $mem\t# Prefetch to level 2 cache for write" %}
  6881   opcode(0x0F, 0x18);     /* Opcode 0F 18 /3 */
  6882   ins_encode(REX_mem(mem), OpcP, OpcS, RM_opc_mem(0x03, mem));
  6883   ins_pipe(ialu_mem);
  6884 %}
  6886 //----------Store Instructions-------------------------------------------------
  6888 // Store Byte
  6889 instruct storeB(memory mem, rRegI src)
  6890 %{
  6891   match(Set mem (StoreB mem src));
  6893   ins_cost(125); // XXX
  6894   format %{ "movb    $mem, $src\t# byte" %}
  6895   opcode(0x88);
  6896   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
  6897   ins_pipe(ialu_mem_reg);
  6898 %}
  6900 // Store Char/Short
  6901 instruct storeC(memory mem, rRegI src)
  6902 %{
  6903   match(Set mem (StoreC mem src));
  6905   ins_cost(125); // XXX
  6906   format %{ "movw    $mem, $src\t# char/short" %}
  6907   opcode(0x89);
  6908   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
  6909   ins_pipe(ialu_mem_reg);
  6910 %}
  6912 // Store Integer
  6913 instruct storeI(memory mem, rRegI src)
  6914 %{
  6915   match(Set mem (StoreI mem src));
  6917   ins_cost(125); // XXX
  6918   format %{ "movl    $mem, $src\t# int" %}
  6919   opcode(0x89);
  6920   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
  6921   ins_pipe(ialu_mem_reg);
  6922 %}
  6924 // Store Long
  6925 instruct storeL(memory mem, rRegL src)
  6926 %{
  6927   match(Set mem (StoreL mem src));
  6929   ins_cost(125); // XXX
  6930   format %{ "movq    $mem, $src\t# long" %}
  6931   opcode(0x89);
  6932   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
  6933   ins_pipe(ialu_mem_reg); // XXX
  6934 %}
  6936 // Store Pointer
  6937 instruct storeP(memory mem, any_RegP src)
  6938 %{
  6939   match(Set mem (StoreP mem src));
  6941   ins_cost(125); // XXX
  6942   format %{ "movq    $mem, $src\t# ptr" %}
  6943   opcode(0x89);
  6944   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
  6945   ins_pipe(ialu_mem_reg);
  6946 %}
  6948 instruct storeImmP0(memory mem, immP0 zero)
  6949 %{
  6950   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
  6951   match(Set mem (StoreP mem zero));
  6953   ins_cost(125); // XXX
  6954   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
  6955   ins_encode %{
  6956     __ movq($mem$$Address, r12);
  6957   %}
  6958   ins_pipe(ialu_mem_reg);
  6959 %}
  6961 // Store NULL Pointer, mark word, or other simple pointer constant.
  6962 instruct storeImmP(memory mem, immP31 src)
  6963 %{
  6964   match(Set mem (StoreP mem src));
  6966   ins_cost(150); // XXX
  6967   format %{ "movq    $mem, $src\t# ptr" %}
  6968   opcode(0xC7); /* C7 /0 */
  6969   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
  6970   ins_pipe(ialu_mem_imm);
  6971 %}
  6973 // Store Compressed Pointer
  6974 instruct storeN(memory mem, rRegN src)
  6975 %{
  6976   match(Set mem (StoreN mem src));
  6978   ins_cost(125); // XXX
  6979   format %{ "movl    $mem, $src\t# compressed ptr" %}
  6980   ins_encode %{
  6981     __ movl($mem$$Address, $src$$Register);
  6982   %}
  6983   ins_pipe(ialu_mem_reg);
  6984 %}
  6986 instruct storeImmN0(memory mem, immN0 zero)
  6987 %{
  6988   predicate(Universe::narrow_oop_base() == NULL);
  6989   match(Set mem (StoreN mem zero));
  6991   ins_cost(125); // XXX
  6992   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
  6993   ins_encode %{
  6994     __ movl($mem$$Address, r12);
  6995   %}
  6996   ins_pipe(ialu_mem_reg);
  6997 %}
  6999 instruct storeImmN(memory mem, immN src)
  7000 %{
  7001   match(Set mem (StoreN mem src));
  7003   ins_cost(150); // XXX
  7004   format %{ "movl    $mem, $src\t# compressed ptr" %}
  7005   ins_encode %{
  7006     address con = (address)$src$$constant;
  7007     if (con == NULL) {
  7008       __ movl($mem$$Address, (int32_t)0);
  7009     } else {
  7010       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
  7012   %}
  7013   ins_pipe(ialu_mem_imm);
  7014 %}
  7016 // Store Integer Immediate
  7017 instruct storeImmI0(memory mem, immI0 zero)
  7018 %{
  7019   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
  7020   match(Set mem (StoreI mem zero));
  7022   ins_cost(125); // XXX
  7023   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
  7024   ins_encode %{
  7025     __ movl($mem$$Address, r12);
  7026   %}
  7027   ins_pipe(ialu_mem_reg);
  7028 %}
  7030 instruct storeImmI(memory mem, immI src)
  7031 %{
  7032   match(Set mem (StoreI mem src));
  7034   ins_cost(150);
  7035   format %{ "movl    $mem, $src\t# int" %}
  7036   opcode(0xC7); /* C7 /0 */
  7037   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
  7038   ins_pipe(ialu_mem_imm);
  7039 %}
  7041 // Store Long Immediate
  7042 instruct storeImmL0(memory mem, immL0 zero)
  7043 %{
  7044   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
  7045   match(Set mem (StoreL mem zero));
  7047   ins_cost(125); // XXX
  7048   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
  7049   ins_encode %{
  7050     __ movq($mem$$Address, r12);
  7051   %}
  7052   ins_pipe(ialu_mem_reg);
  7053 %}
  7055 instruct storeImmL(memory mem, immL32 src)
  7056 %{
  7057   match(Set mem (StoreL mem src));
  7059   ins_cost(150);
  7060   format %{ "movq    $mem, $src\t# long" %}
  7061   opcode(0xC7); /* C7 /0 */
  7062   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
  7063   ins_pipe(ialu_mem_imm);
  7064 %}
  7066 // Store Short/Char Immediate
  7067 instruct storeImmC0(memory mem, immI0 zero)
  7068 %{
  7069   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
  7070   match(Set mem (StoreC mem zero));
  7072   ins_cost(125); // XXX
  7073   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
  7074   ins_encode %{
  7075     __ movw($mem$$Address, r12);
  7076   %}
  7077   ins_pipe(ialu_mem_reg);
  7078 %}
  7080 instruct storeImmI16(memory mem, immI16 src)
  7081 %{
  7082   predicate(UseStoreImmI16);
  7083   match(Set mem (StoreC mem src));
  7085   ins_cost(150);
  7086   format %{ "movw    $mem, $src\t# short/char" %}
  7087   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
  7088   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
  7089   ins_pipe(ialu_mem_imm);
  7090 %}
  7092 // Store Byte Immediate
  7093 instruct storeImmB0(memory mem, immI0 zero)
  7094 %{
  7095   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
  7096   match(Set mem (StoreB mem zero));
  7098   ins_cost(125); // XXX
  7099   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
  7100   ins_encode %{
  7101     __ movb($mem$$Address, r12);
  7102   %}
  7103   ins_pipe(ialu_mem_reg);
  7104 %}
  7106 instruct storeImmB(memory mem, immI8 src)
  7107 %{
  7108   match(Set mem (StoreB mem src));
  7110   ins_cost(150); // XXX
  7111   format %{ "movb    $mem, $src\t# byte" %}
  7112   opcode(0xC6); /* C6 /0 */
  7113   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
  7114   ins_pipe(ialu_mem_imm);
  7115 %}
  7117 // Store Aligned Packed Byte XMM register to memory
  7118 instruct storeA8B(memory mem, regD src) %{
  7119   match(Set mem (Store8B mem src));
  7120   ins_cost(145);
  7121   format %{ "MOVQ  $mem,$src\t! packed8B" %}
  7122   ins_encode( movq_st(mem, src));
  7123   ins_pipe( pipe_slow );
  7124 %}
  7126 // Store Aligned Packed Char/Short XMM register to memory
  7127 instruct storeA4C(memory mem, regD src) %{
  7128   match(Set mem (Store4C mem src));
  7129   ins_cost(145);
  7130   format %{ "MOVQ  $mem,$src\t! packed4C" %}
  7131   ins_encode( movq_st(mem, src));
  7132   ins_pipe( pipe_slow );
  7133 %}
  7135 // Store Aligned Packed Integer XMM register to memory
  7136 instruct storeA2I(memory mem, regD src) %{
  7137   match(Set mem (Store2I mem src));
  7138   ins_cost(145);
  7139   format %{ "MOVQ  $mem,$src\t! packed2I" %}
  7140   ins_encode( movq_st(mem, src));
  7141   ins_pipe( pipe_slow );
  7142 %}
  7144 // Store CMS card-mark Immediate
  7145 instruct storeImmCM0_reg(memory mem, immI0 zero)
  7146 %{
  7147   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
  7148   match(Set mem (StoreCM mem zero));
  7150   ins_cost(125); // XXX
  7151   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
  7152   ins_encode %{
  7153     __ movb($mem$$Address, r12);
  7154   %}
  7155   ins_pipe(ialu_mem_reg);
  7156 %}
  7158 instruct storeImmCM0(memory mem, immI0 src)
  7159 %{
  7160   match(Set mem (StoreCM mem src));
  7162   ins_cost(150); // XXX
  7163   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
  7164   opcode(0xC6); /* C6 /0 */
  7165   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
  7166   ins_pipe(ialu_mem_imm);
  7167 %}
  7169 // Store Aligned Packed Single Float XMM register to memory
  7170 instruct storeA2F(memory mem, regD src) %{
  7171   match(Set mem (Store2F mem src));
  7172   ins_cost(145);
  7173   format %{ "MOVQ  $mem,$src\t! packed2F" %}
  7174   ins_encode( movq_st(mem, src));
  7175   ins_pipe( pipe_slow );
  7176 %}
  7178 // Store Float
  7179 instruct storeF(memory mem, regF src)
  7180 %{
  7181   match(Set mem (StoreF mem src));
  7183   ins_cost(95); // XXX
  7184   format %{ "movss   $mem, $src\t# float" %}
  7185   opcode(0xF3, 0x0F, 0x11);
  7186   ins_encode(OpcP, REX_reg_mem(src, mem), OpcS, OpcT, reg_mem(src, mem));
  7187   ins_pipe(pipe_slow); // XXX
  7188 %}
  7190 // Store immediate Float value (it is faster than store from XMM register)
  7191 instruct storeF0(memory mem, immF0 zero)
  7192 %{
  7193   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
  7194   match(Set mem (StoreF mem zero));
  7196   ins_cost(25); // XXX
  7197   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
  7198   ins_encode %{
  7199     __ movl($mem$$Address, r12);
  7200   %}
  7201   ins_pipe(ialu_mem_reg);
  7202 %}
  7204 instruct storeF_imm(memory mem, immF src)
  7205 %{
  7206   match(Set mem (StoreF mem src));
  7208   ins_cost(50);
  7209   format %{ "movl    $mem, $src\t# float" %}
  7210   opcode(0xC7); /* C7 /0 */
  7211   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
  7212   ins_pipe(ialu_mem_imm);
  7213 %}
  7215 // Store Double
  7216 instruct storeD(memory mem, regD src)
  7217 %{
  7218   match(Set mem (StoreD mem src));
  7220   ins_cost(95); // XXX
  7221   format %{ "movsd   $mem, $src\t# double" %}
  7222   opcode(0xF2, 0x0F, 0x11);
  7223   ins_encode(OpcP, REX_reg_mem(src, mem), OpcS, OpcT, reg_mem(src, mem));
  7224   ins_pipe(pipe_slow); // XXX
  7225 %}
  7227 // Store immediate double 0.0 (it is faster than store from XMM register)
  7228 instruct storeD0_imm(memory mem, immD0 src)
  7229 %{
  7230   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
  7231   match(Set mem (StoreD mem src));
  7233   ins_cost(50);
  7234   format %{ "movq    $mem, $src\t# double 0." %}
  7235   opcode(0xC7); /* C7 /0 */
  7236   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
  7237   ins_pipe(ialu_mem_imm);
  7238 %}
  7240 instruct storeD0(memory mem, immD0 zero)
  7241 %{
  7242   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
  7243   match(Set mem (StoreD mem zero));
  7245   ins_cost(25); // XXX
  7246   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
  7247   ins_encode %{
  7248     __ movq($mem$$Address, r12);
  7249   %}
  7250   ins_pipe(ialu_mem_reg);
  7251 %}
  7253 instruct storeSSI(stackSlotI dst, rRegI src)
  7254 %{
  7255   match(Set dst src);
  7257   ins_cost(100);
  7258   format %{ "movl    $dst, $src\t# int stk" %}
  7259   opcode(0x89);
  7260   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
  7261   ins_pipe( ialu_mem_reg );
  7262 %}
  7264 instruct storeSSL(stackSlotL dst, rRegL src)
  7265 %{
  7266   match(Set dst src);
  7268   ins_cost(100);
  7269   format %{ "movq    $dst, $src\t# long stk" %}
  7270   opcode(0x89);
  7271   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
  7272   ins_pipe(ialu_mem_reg);
  7273 %}
  7275 instruct storeSSP(stackSlotP dst, rRegP src)
  7276 %{
  7277   match(Set dst src);
  7279   ins_cost(100);
  7280   format %{ "movq    $dst, $src\t# ptr stk" %}
  7281   opcode(0x89);
  7282   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
  7283   ins_pipe(ialu_mem_reg);
  7284 %}
  7286 instruct storeSSF(stackSlotF dst, regF src)
  7287 %{
  7288   match(Set dst src);
  7290   ins_cost(95); // XXX
  7291   format %{ "movss   $dst, $src\t# float stk" %}
  7292   opcode(0xF3, 0x0F, 0x11);
  7293   ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
  7294   ins_pipe(pipe_slow); // XXX
  7295 %}
  7297 instruct storeSSD(stackSlotD dst, regD src)
  7298 %{
  7299   match(Set dst src);
  7301   ins_cost(95); // XXX
  7302   format %{ "movsd   $dst, $src\t# double stk" %}
  7303   opcode(0xF2, 0x0F, 0x11);
  7304   ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
  7305   ins_pipe(pipe_slow); // XXX
  7306 %}
  7308 //----------BSWAP Instructions-------------------------------------------------
  7309 instruct bytes_reverse_int(rRegI dst) %{
  7310   match(Set dst (ReverseBytesI dst));
  7312   format %{ "bswapl  $dst" %}
  7313   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
  7314   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
  7315   ins_pipe( ialu_reg );
  7316 %}
  7318 instruct bytes_reverse_long(rRegL dst) %{
  7319   match(Set dst (ReverseBytesL dst));
  7321   format %{ "bswapq  $dst" %}
  7323   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
  7324   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
  7325   ins_pipe( ialu_reg);
  7326 %}
  7328 instruct bytes_reverse_unsigned_short(rRegI dst) %{
  7329   match(Set dst (ReverseBytesUS dst));
  7331   format %{ "bswapl  $dst\n\t" 
  7332             "shrl    $dst,16\n\t" %}
  7333   ins_encode %{
  7334     __ bswapl($dst$$Register);
  7335     __ shrl($dst$$Register, 16); 
  7336   %}
  7337   ins_pipe( ialu_reg );
  7338 %}
  7340 instruct bytes_reverse_short(rRegI dst) %{
  7341   match(Set dst (ReverseBytesS dst));
  7343   format %{ "bswapl  $dst\n\t" 
  7344             "sar     $dst,16\n\t" %}
  7345   ins_encode %{
  7346     __ bswapl($dst$$Register);
  7347     __ sarl($dst$$Register, 16); 
  7348   %}
  7349   ins_pipe( ialu_reg );
  7350 %}
  7352 //---------- Zeros Count Instructions ------------------------------------------
  7354 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
  7355   predicate(UseCountLeadingZerosInstruction);
  7356   match(Set dst (CountLeadingZerosI src));
  7357   effect(KILL cr);
  7359   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
  7360   ins_encode %{
  7361     __ lzcntl($dst$$Register, $src$$Register);
  7362   %}
  7363   ins_pipe(ialu_reg);
  7364 %}
  7366 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
  7367   predicate(!UseCountLeadingZerosInstruction);
  7368   match(Set dst (CountLeadingZerosI src));
  7369   effect(KILL cr);
  7371   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
  7372             "jnz     skip\n\t"
  7373             "movl    $dst, -1\n"
  7374       "skip:\n\t"
  7375             "negl    $dst\n\t"
  7376             "addl    $dst, 31" %}
  7377   ins_encode %{
  7378     Register Rdst = $dst$$Register;
  7379     Register Rsrc = $src$$Register;
  7380     Label skip;
  7381     __ bsrl(Rdst, Rsrc);
  7382     __ jccb(Assembler::notZero, skip);
  7383     __ movl(Rdst, -1);
  7384     __ bind(skip);
  7385     __ negl(Rdst);
  7386     __ addl(Rdst, BitsPerInt - 1);
  7387   %}
  7388   ins_pipe(ialu_reg);
  7389 %}
  7391 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
  7392   predicate(UseCountLeadingZerosInstruction);
  7393   match(Set dst (CountLeadingZerosL src));
  7394   effect(KILL cr);
  7396   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
  7397   ins_encode %{
  7398     __ lzcntq($dst$$Register, $src$$Register);
  7399   %}
  7400   ins_pipe(ialu_reg);
  7401 %}
  7403 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
  7404   predicate(!UseCountLeadingZerosInstruction);
  7405   match(Set dst (CountLeadingZerosL src));
  7406   effect(KILL cr);
  7408   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
  7409             "jnz     skip\n\t"
  7410             "movl    $dst, -1\n"
  7411       "skip:\n\t"
  7412             "negl    $dst\n\t"
  7413             "addl    $dst, 63" %}
  7414   ins_encode %{
  7415     Register Rdst = $dst$$Register;
  7416     Register Rsrc = $src$$Register;
  7417     Label skip;
  7418     __ bsrq(Rdst, Rsrc);
  7419     __ jccb(Assembler::notZero, skip);
  7420     __ movl(Rdst, -1);
  7421     __ bind(skip);
  7422     __ negl(Rdst);
  7423     __ addl(Rdst, BitsPerLong - 1);
  7424   %}
  7425   ins_pipe(ialu_reg);
  7426 %}
  7428 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
  7429   match(Set dst (CountTrailingZerosI src));
  7430   effect(KILL cr);
  7432   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
  7433             "jnz     done\n\t"
  7434             "movl    $dst, 32\n"
  7435       "done:" %}
  7436   ins_encode %{
  7437     Register Rdst = $dst$$Register;
  7438     Label done;
  7439     __ bsfl(Rdst, $src$$Register);
  7440     __ jccb(Assembler::notZero, done);
  7441     __ movl(Rdst, BitsPerInt);
  7442     __ bind(done);
  7443   %}
  7444   ins_pipe(ialu_reg);
  7445 %}
  7447 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
  7448   match(Set dst (CountTrailingZerosL src));
  7449   effect(KILL cr);
  7451   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
  7452             "jnz     done\n\t"
  7453             "movl    $dst, 64\n"
  7454       "done:" %}
  7455   ins_encode %{
  7456     Register Rdst = $dst$$Register;
  7457     Label done;
  7458     __ bsfq(Rdst, $src$$Register);
  7459     __ jccb(Assembler::notZero, done);
  7460     __ movl(Rdst, BitsPerLong);
  7461     __ bind(done);
  7462   %}
  7463   ins_pipe(ialu_reg);
  7464 %}
  7467 //---------- Population Count Instructions -------------------------------------
  7469 instruct popCountI(rRegI dst, rRegI src) %{
  7470   predicate(UsePopCountInstruction);
  7471   match(Set dst (PopCountI src));
  7473   format %{ "popcnt  $dst, $src" %}
  7474   ins_encode %{
  7475     __ popcntl($dst$$Register, $src$$Register);
  7476   %}
  7477   ins_pipe(ialu_reg);
  7478 %}
  7480 instruct popCountI_mem(rRegI dst, memory mem) %{
  7481   predicate(UsePopCountInstruction);
  7482   match(Set dst (PopCountI (LoadI mem)));
  7484   format %{ "popcnt  $dst, $mem" %}
  7485   ins_encode %{
  7486     __ popcntl($dst$$Register, $mem$$Address);
  7487   %}
  7488   ins_pipe(ialu_reg);
  7489 %}
  7491 // Note: Long.bitCount(long) returns an int.
  7492 instruct popCountL(rRegI dst, rRegL src) %{
  7493   predicate(UsePopCountInstruction);
  7494   match(Set dst (PopCountL src));
  7496   format %{ "popcnt  $dst, $src" %}
  7497   ins_encode %{
  7498     __ popcntq($dst$$Register, $src$$Register);
  7499   %}
  7500   ins_pipe(ialu_reg);
  7501 %}
  7503 // Note: Long.bitCount(long) returns an int.
  7504 instruct popCountL_mem(rRegI dst, memory mem) %{
  7505   predicate(UsePopCountInstruction);
  7506   match(Set dst (PopCountL (LoadL mem)));
  7508   format %{ "popcnt  $dst, $mem" %}
  7509   ins_encode %{
  7510     __ popcntq($dst$$Register, $mem$$Address);
  7511   %}
  7512   ins_pipe(ialu_reg);
  7513 %}
  7516 //----------MemBar Instructions-----------------------------------------------
  7517 // Memory barrier flavors
  7519 instruct membar_acquire()
  7520 %{
  7521   match(MemBarAcquire);
  7522   ins_cost(0);
  7524   size(0);
  7525   format %{ "MEMBAR-acquire ! (empty encoding)" %}
  7526   ins_encode();
  7527   ins_pipe(empty);
  7528 %}
  7530 instruct membar_acquire_lock()
  7531 %{
  7532   match(MemBarAcquire);
  7533   predicate(Matcher::prior_fast_lock(n));
  7534   ins_cost(0);
  7536   size(0);
  7537   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
  7538   ins_encode();
  7539   ins_pipe(empty);
  7540 %}
  7542 instruct membar_release()
  7543 %{
  7544   match(MemBarRelease);
  7545   ins_cost(0);
  7547   size(0);
  7548   format %{ "MEMBAR-release ! (empty encoding)" %}
  7549   ins_encode();
  7550   ins_pipe(empty);
  7551 %}
  7553 instruct membar_release_lock()
  7554 %{
  7555   match(MemBarRelease);
  7556   predicate(Matcher::post_fast_unlock(n));
  7557   ins_cost(0);
  7559   size(0);
  7560   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
  7561   ins_encode();
  7562   ins_pipe(empty);
  7563 %}
  7565 instruct membar_volatile(rFlagsReg cr) %{
  7566   match(MemBarVolatile);
  7567   effect(KILL cr);
  7568   ins_cost(400);
  7570   format %{ 
  7571     $$template
  7572     if (os::is_MP()) {
  7573       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
  7574     } else {
  7575       $$emit$$"MEMBAR-volatile ! (empty encoding)"
  7577   %}
  7578   ins_encode %{
  7579     __ membar(Assembler::StoreLoad);
  7580   %}
  7581   ins_pipe(pipe_slow);
  7582 %}
  7584 instruct unnecessary_membar_volatile()
  7585 %{
  7586   match(MemBarVolatile);
  7587   predicate(Matcher::post_store_load_barrier(n));
  7588   ins_cost(0);
  7590   size(0);
  7591   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
  7592   ins_encode();
  7593   ins_pipe(empty);
  7594 %}
  7596 //----------Move Instructions--------------------------------------------------
  7598 instruct castX2P(rRegP dst, rRegL src)
  7599 %{
  7600   match(Set dst (CastX2P src));
  7602   format %{ "movq    $dst, $src\t# long->ptr" %}
  7603   ins_encode(enc_copy_wide(dst, src));
  7604   ins_pipe(ialu_reg_reg); // XXX
  7605 %}
  7607 instruct castP2X(rRegL dst, rRegP src)
  7608 %{
  7609   match(Set dst (CastP2X src));
  7611   format %{ "movq    $dst, $src\t# ptr -> long" %}
  7612   ins_encode(enc_copy_wide(dst, src));
  7613   ins_pipe(ialu_reg_reg); // XXX
  7614 %}
  7617 // Convert oop pointer into compressed form
  7618 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
  7619   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
  7620   match(Set dst (EncodeP src));
  7621   effect(KILL cr);
  7622   format %{ "encode_heap_oop $dst,$src" %}
  7623   ins_encode %{
  7624     Register s = $src$$Register;
  7625     Register d = $dst$$Register;
  7626     if (s != d) {
  7627       __ movq(d, s);
  7629     __ encode_heap_oop(d);
  7630   %}
  7631   ins_pipe(ialu_reg_long);
  7632 %}
  7634 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
  7635   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
  7636   match(Set dst (EncodeP src));
  7637   effect(KILL cr);
  7638   format %{ "encode_heap_oop_not_null $dst,$src" %}
  7639   ins_encode %{
  7640     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
  7641   %}
  7642   ins_pipe(ialu_reg_long);
  7643 %}
  7645 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
  7646   predicate(n->bottom_type()->is_oopptr()->ptr() != TypePtr::NotNull &&
  7647             n->bottom_type()->is_oopptr()->ptr() != TypePtr::Constant);
  7648   match(Set dst (DecodeN src));
  7649   effect(KILL cr);
  7650   format %{ "decode_heap_oop $dst,$src" %}
  7651   ins_encode %{
  7652     Register s = $src$$Register;
  7653     Register d = $dst$$Register;
  7654     if (s != d) {
  7655       __ movq(d, s);
  7657     __ decode_heap_oop(d);
  7658   %}
  7659   ins_pipe(ialu_reg_long);
  7660 %}
  7662 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
  7663   predicate(n->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull ||
  7664             n->bottom_type()->is_oopptr()->ptr() == TypePtr::Constant);
  7665   match(Set dst (DecodeN src));
  7666   effect(KILL cr);
  7667   format %{ "decode_heap_oop_not_null $dst,$src" %}
  7668   ins_encode %{
  7669     Register s = $src$$Register;
  7670     Register d = $dst$$Register;
  7671     if (s != d) {
  7672       __ decode_heap_oop_not_null(d, s);
  7673     } else {
  7674       __ decode_heap_oop_not_null(d);
  7676   %}
  7677   ins_pipe(ialu_reg_long);
  7678 %}
  7681 //----------Conditional Move---------------------------------------------------
  7682 // Jump
  7683 // dummy instruction for generating temp registers
  7684 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
  7685   match(Jump (LShiftL switch_val shift));
  7686   ins_cost(350);
  7687   predicate(false);
  7688   effect(TEMP dest);
  7690   format %{ "leaq    $dest, table_base\n\t"
  7691             "jmp     [$dest + $switch_val << $shift]\n\t" %}
  7692   ins_encode(jump_enc_offset(switch_val, shift, dest));
  7693   ins_pipe(pipe_jmp);
  7694   ins_pc_relative(1);
  7695 %}
  7697 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
  7698   match(Jump (AddL (LShiftL switch_val shift) offset));
  7699   ins_cost(350);
  7700   effect(TEMP dest);
  7702   format %{ "leaq    $dest, table_base\n\t"
  7703             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
  7704   ins_encode(jump_enc_addr(switch_val, shift, offset, dest));
  7705   ins_pipe(pipe_jmp);
  7706   ins_pc_relative(1);
  7707 %}
  7709 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
  7710   match(Jump switch_val);
  7711   ins_cost(350);
  7712   effect(TEMP dest);
  7714   format %{ "leaq    $dest, table_base\n\t"
  7715             "jmp     [$dest + $switch_val]\n\t" %}
  7716   ins_encode(jump_enc(switch_val, dest));
  7717   ins_pipe(pipe_jmp);
  7718   ins_pc_relative(1);
  7719 %}
  7721 // Conditional move
  7722 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
  7723 %{
  7724   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
  7726   ins_cost(200); // XXX
  7727   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
  7728   opcode(0x0F, 0x40);
  7729   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
  7730   ins_pipe(pipe_cmov_reg);
  7731 %}
  7733 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
  7734   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
  7736   ins_cost(200); // XXX
  7737   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
  7738   opcode(0x0F, 0x40);
  7739   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
  7740   ins_pipe(pipe_cmov_reg);
  7741 %}
  7743 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
  7744   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
  7745   ins_cost(200);
  7746   expand %{
  7747     cmovI_regU(cop, cr, dst, src);
  7748   %}
  7749 %}
  7751 // Conditional move
  7752 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
  7753   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
  7755   ins_cost(250); // XXX
  7756   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
  7757   opcode(0x0F, 0x40);
  7758   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
  7759   ins_pipe(pipe_cmov_mem);
  7760 %}
  7762 // Conditional move
  7763 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
  7764 %{
  7765   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
  7767   ins_cost(250); // XXX
  7768   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
  7769   opcode(0x0F, 0x40);
  7770   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
  7771   ins_pipe(pipe_cmov_mem);
  7772 %}
  7774 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
  7775   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
  7776   ins_cost(250);
  7777   expand %{
  7778     cmovI_memU(cop, cr, dst, src);
  7779   %}
  7780 %}
  7782 // Conditional move
  7783 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
  7784 %{
  7785   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
  7787   ins_cost(200); // XXX
  7788   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
  7789   opcode(0x0F, 0x40);
  7790   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
  7791   ins_pipe(pipe_cmov_reg);
  7792 %}
  7794 // Conditional move
  7795 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
  7796 %{
  7797   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
  7799   ins_cost(200); // XXX
  7800   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
  7801   opcode(0x0F, 0x40);
  7802   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
  7803   ins_pipe(pipe_cmov_reg);
  7804 %}
  7806 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
  7807   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
  7808   ins_cost(200);
  7809   expand %{
  7810     cmovN_regU(cop, cr, dst, src);
  7811   %}
  7812 %}
  7814 // Conditional move
  7815 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
  7816 %{
  7817   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
  7819   ins_cost(200); // XXX
  7820   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
  7821   opcode(0x0F, 0x40);
  7822   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
  7823   ins_pipe(pipe_cmov_reg);  // XXX
  7824 %}
  7826 // Conditional move
  7827 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
  7828 %{
  7829   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
  7831   ins_cost(200); // XXX
  7832   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
  7833   opcode(0x0F, 0x40);
  7834   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
  7835   ins_pipe(pipe_cmov_reg); // XXX
  7836 %}
  7838 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
  7839   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
  7840   ins_cost(200);
  7841   expand %{
  7842     cmovP_regU(cop, cr, dst, src);
  7843   %}
  7844 %}
  7846 // DISABLED: Requires the ADLC to emit a bottom_type call that
  7847 // correctly meets the two pointer arguments; one is an incoming
  7848 // register but the other is a memory operand.  ALSO appears to
  7849 // be buggy with implicit null checks.
  7850 //
  7851 //// Conditional move
  7852 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
  7853 //%{
  7854 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
  7855 //  ins_cost(250);
  7856 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
  7857 //  opcode(0x0F,0x40);
  7858 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
  7859 //  ins_pipe( pipe_cmov_mem );
  7860 //%}
  7861 //
  7862 //// Conditional move
  7863 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
  7864 //%{
  7865 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
  7866 //  ins_cost(250);
  7867 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
  7868 //  opcode(0x0F,0x40);
  7869 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
  7870 //  ins_pipe( pipe_cmov_mem );
  7871 //%}
  7873 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
  7874 %{
  7875   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
  7877   ins_cost(200); // XXX
  7878   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
  7879   opcode(0x0F, 0x40);
  7880   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
  7881   ins_pipe(pipe_cmov_reg);  // XXX
  7882 %}
  7884 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
  7885 %{
  7886   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
  7888   ins_cost(200); // XXX
  7889   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
  7890   opcode(0x0F, 0x40);
  7891   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
  7892   ins_pipe(pipe_cmov_mem);  // XXX
  7893 %}
  7895 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
  7896 %{
  7897   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
  7899   ins_cost(200); // XXX
  7900   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
  7901   opcode(0x0F, 0x40);
  7902   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
  7903   ins_pipe(pipe_cmov_reg); // XXX
  7904 %}
  7906 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
  7907   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
  7908   ins_cost(200);
  7909   expand %{
  7910     cmovL_regU(cop, cr, dst, src);
  7911   %}
  7912 %}
  7914 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
  7915 %{
  7916   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
  7918   ins_cost(200); // XXX
  7919   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
  7920   opcode(0x0F, 0x40);
  7921   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
  7922   ins_pipe(pipe_cmov_mem); // XXX
  7923 %}
  7925 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
  7926   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
  7927   ins_cost(200);
  7928   expand %{
  7929     cmovL_memU(cop, cr, dst, src);
  7930   %}
  7931 %}
  7933 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
  7934 %{
  7935   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
  7937   ins_cost(200); // XXX
  7938   format %{ "jn$cop    skip\t# signed cmove float\n\t"
  7939             "movss     $dst, $src\n"
  7940     "skip:" %}
  7941   ins_encode(enc_cmovf_branch(cop, dst, src));
  7942   ins_pipe(pipe_slow);
  7943 %}
  7945 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
  7946 // %{
  7947 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
  7949 //   ins_cost(200); // XXX
  7950 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
  7951 //             "movss     $dst, $src\n"
  7952 //     "skip:" %}
  7953 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
  7954 //   ins_pipe(pipe_slow);
  7955 // %}
  7957 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
  7958 %{
  7959   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
  7961   ins_cost(200); // XXX
  7962   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
  7963             "movss     $dst, $src\n"
  7964     "skip:" %}
  7965   ins_encode(enc_cmovf_branch(cop, dst, src));
  7966   ins_pipe(pipe_slow);
  7967 %}
  7969 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
  7970   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
  7971   ins_cost(200);
  7972   expand %{
  7973     cmovF_regU(cop, cr, dst, src);
  7974   %}
  7975 %}
  7977 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
  7978 %{
  7979   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
  7981   ins_cost(200); // XXX
  7982   format %{ "jn$cop    skip\t# signed cmove double\n\t"
  7983             "movsd     $dst, $src\n"
  7984     "skip:" %}
  7985   ins_encode(enc_cmovd_branch(cop, dst, src));
  7986   ins_pipe(pipe_slow);
  7987 %}
  7989 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
  7990 %{
  7991   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
  7993   ins_cost(200); // XXX
  7994   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
  7995             "movsd     $dst, $src\n"
  7996     "skip:" %}
  7997   ins_encode(enc_cmovd_branch(cop, dst, src));
  7998   ins_pipe(pipe_slow);
  7999 %}
  8001 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
  8002   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
  8003   ins_cost(200);
  8004   expand %{
  8005     cmovD_regU(cop, cr, dst, src);
  8006   %}
  8007 %}
  8009 //----------Arithmetic Instructions--------------------------------------------
  8010 //----------Addition Instructions----------------------------------------------
  8012 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
  8013 %{
  8014   match(Set dst (AddI dst src));
  8015   effect(KILL cr);
  8017   format %{ "addl    $dst, $src\t# int" %}
  8018   opcode(0x03);
  8019   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
  8020   ins_pipe(ialu_reg_reg);
  8021 %}
  8023 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
  8024 %{
  8025   match(Set dst (AddI dst src));
  8026   effect(KILL cr);
  8028   format %{ "addl    $dst, $src\t# int" %}
  8029   opcode(0x81, 0x00); /* /0 id */
  8030   ins_encode(OpcSErm(dst, src), Con8or32(src));
  8031   ins_pipe( ialu_reg );
  8032 %}
  8034 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
  8035 %{
  8036   match(Set dst (AddI dst (LoadI src)));
  8037   effect(KILL cr);
  8039   ins_cost(125); // XXX
  8040   format %{ "addl    $dst, $src\t# int" %}
  8041   opcode(0x03);
  8042   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
  8043   ins_pipe(ialu_reg_mem);
  8044 %}
  8046 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
  8047 %{
  8048   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
  8049   effect(KILL cr);
  8051   ins_cost(150); // XXX
  8052   format %{ "addl    $dst, $src\t# int" %}
  8053   opcode(0x01); /* Opcode 01 /r */
  8054   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
  8055   ins_pipe(ialu_mem_reg);
  8056 %}
  8058 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
  8059 %{
  8060   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
  8061   effect(KILL cr);
  8063   ins_cost(125); // XXX
  8064   format %{ "addl    $dst, $src\t# int" %}
  8065   opcode(0x81); /* Opcode 81 /0 id */
  8066   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
  8067   ins_pipe(ialu_mem_imm);
  8068 %}
  8070 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
  8071 %{
  8072   predicate(UseIncDec);
  8073   match(Set dst (AddI dst src));
  8074   effect(KILL cr);
  8076   format %{ "incl    $dst\t# int" %}
  8077   opcode(0xFF, 0x00); // FF /0
  8078   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  8079   ins_pipe(ialu_reg);
  8080 %}
  8082 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
  8083 %{
  8084   predicate(UseIncDec);
  8085   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
  8086   effect(KILL cr);
  8088   ins_cost(125); // XXX
  8089   format %{ "incl    $dst\t# int" %}
  8090   opcode(0xFF); /* Opcode FF /0 */
  8091   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
  8092   ins_pipe(ialu_mem_imm);
  8093 %}
  8095 // XXX why does that use AddI
  8096 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
  8097 %{
  8098   predicate(UseIncDec);
  8099   match(Set dst (AddI dst src));
  8100   effect(KILL cr);
  8102   format %{ "decl    $dst\t# int" %}
  8103   opcode(0xFF, 0x01); // FF /1
  8104   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  8105   ins_pipe(ialu_reg);
  8106 %}
  8108 // XXX why does that use AddI
  8109 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
  8110 %{
  8111   predicate(UseIncDec);
  8112   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
  8113   effect(KILL cr);
  8115   ins_cost(125); // XXX
  8116   format %{ "decl    $dst\t# int" %}
  8117   opcode(0xFF); /* Opcode FF /1 */
  8118   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
  8119   ins_pipe(ialu_mem_imm);
  8120 %}
  8122 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
  8123 %{
  8124   match(Set dst (AddI src0 src1));
  8126   ins_cost(110);
  8127   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
  8128   opcode(0x8D); /* 0x8D /r */
  8129   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
  8130   ins_pipe(ialu_reg_reg);
  8131 %}
  8133 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
  8134 %{
  8135   match(Set dst (AddL dst src));
  8136   effect(KILL cr);
  8138   format %{ "addq    $dst, $src\t# long" %}
  8139   opcode(0x03);
  8140   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
  8141   ins_pipe(ialu_reg_reg);
  8142 %}
  8144 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
  8145 %{
  8146   match(Set dst (AddL dst src));
  8147   effect(KILL cr);
  8149   format %{ "addq    $dst, $src\t# long" %}
  8150   opcode(0x81, 0x00); /* /0 id */
  8151   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
  8152   ins_pipe( ialu_reg );
  8153 %}
  8155 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
  8156 %{
  8157   match(Set dst (AddL dst (LoadL src)));
  8158   effect(KILL cr);
  8160   ins_cost(125); // XXX
  8161   format %{ "addq    $dst, $src\t# long" %}
  8162   opcode(0x03);
  8163   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
  8164   ins_pipe(ialu_reg_mem);
  8165 %}
  8167 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
  8168 %{
  8169   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
  8170   effect(KILL cr);
  8172   ins_cost(150); // XXX
  8173   format %{ "addq    $dst, $src\t# long" %}
  8174   opcode(0x01); /* Opcode 01 /r */
  8175   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
  8176   ins_pipe(ialu_mem_reg);
  8177 %}
  8179 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
  8180 %{
  8181   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
  8182   effect(KILL cr);
  8184   ins_cost(125); // XXX
  8185   format %{ "addq    $dst, $src\t# long" %}
  8186   opcode(0x81); /* Opcode 81 /0 id */
  8187   ins_encode(REX_mem_wide(dst),
  8188              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
  8189   ins_pipe(ialu_mem_imm);
  8190 %}
  8192 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
  8193 %{
  8194   predicate(UseIncDec);
  8195   match(Set dst (AddL dst src));
  8196   effect(KILL cr);
  8198   format %{ "incq    $dst\t# long" %}
  8199   opcode(0xFF, 0x00); // FF /0
  8200   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  8201   ins_pipe(ialu_reg);
  8202 %}
  8204 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
  8205 %{
  8206   predicate(UseIncDec);
  8207   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
  8208   effect(KILL cr);
  8210   ins_cost(125); // XXX
  8211   format %{ "incq    $dst\t# long" %}
  8212   opcode(0xFF); /* Opcode FF /0 */
  8213   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
  8214   ins_pipe(ialu_mem_imm);
  8215 %}
  8217 // XXX why does that use AddL
  8218 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
  8219 %{
  8220   predicate(UseIncDec);
  8221   match(Set dst (AddL dst src));
  8222   effect(KILL cr);
  8224   format %{ "decq    $dst\t# long" %}
  8225   opcode(0xFF, 0x01); // FF /1
  8226   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  8227   ins_pipe(ialu_reg);
  8228 %}
  8230 // XXX why does that use AddL
  8231 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
  8232 %{
  8233   predicate(UseIncDec);
  8234   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
  8235   effect(KILL cr);
  8237   ins_cost(125); // XXX
  8238   format %{ "decq    $dst\t# long" %}
  8239   opcode(0xFF); /* Opcode FF /1 */
  8240   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
  8241   ins_pipe(ialu_mem_imm);
  8242 %}
  8244 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
  8245 %{
  8246   match(Set dst (AddL src0 src1));
  8248   ins_cost(110);
  8249   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
  8250   opcode(0x8D); /* 0x8D /r */
  8251   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
  8252   ins_pipe(ialu_reg_reg);
  8253 %}
  8255 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
  8256 %{
  8257   match(Set dst (AddP dst src));
  8258   effect(KILL cr);
  8260   format %{ "addq    $dst, $src\t# ptr" %}
  8261   opcode(0x03);
  8262   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
  8263   ins_pipe(ialu_reg_reg);
  8264 %}
  8266 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
  8267 %{
  8268   match(Set dst (AddP dst src));
  8269   effect(KILL cr);
  8271   format %{ "addq    $dst, $src\t# ptr" %}
  8272   opcode(0x81, 0x00); /* /0 id */
  8273   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
  8274   ins_pipe( ialu_reg );
  8275 %}
  8277 // XXX addP mem ops ????
  8279 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
  8280 %{
  8281   match(Set dst (AddP src0 src1));
  8283   ins_cost(110);
  8284   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
  8285   opcode(0x8D); /* 0x8D /r */
  8286   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
  8287   ins_pipe(ialu_reg_reg);
  8288 %}
  8290 instruct checkCastPP(rRegP dst)
  8291 %{
  8292   match(Set dst (CheckCastPP dst));
  8294   size(0);
  8295   format %{ "# checkcastPP of $dst" %}
  8296   ins_encode(/* empty encoding */);
  8297   ins_pipe(empty);
  8298 %}
  8300 instruct castPP(rRegP dst)
  8301 %{
  8302   match(Set dst (CastPP dst));
  8304   size(0);
  8305   format %{ "# castPP of $dst" %}
  8306   ins_encode(/* empty encoding */);
  8307   ins_pipe(empty);
  8308 %}
  8310 instruct castII(rRegI dst)
  8311 %{
  8312   match(Set dst (CastII dst));
  8314   size(0);
  8315   format %{ "# castII of $dst" %}
  8316   ins_encode(/* empty encoding */);
  8317   ins_cost(0);
  8318   ins_pipe(empty);
  8319 %}
  8321 // LoadP-locked same as a regular LoadP when used with compare-swap
  8322 instruct loadPLocked(rRegP dst, memory mem)
  8323 %{
  8324   match(Set dst (LoadPLocked mem));
  8326   ins_cost(125); // XXX
  8327   format %{ "movq    $dst, $mem\t# ptr locked" %}
  8328   opcode(0x8B);
  8329   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  8330   ins_pipe(ialu_reg_mem); // XXX
  8331 %}
  8333 // LoadL-locked - same as a regular LoadL when used with compare-swap
  8334 instruct loadLLocked(rRegL dst, memory mem)
  8335 %{
  8336   match(Set dst (LoadLLocked mem));
  8338   ins_cost(125); // XXX
  8339   format %{ "movq    $dst, $mem\t# long locked" %}
  8340   opcode(0x8B);
  8341   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
  8342   ins_pipe(ialu_reg_mem); // XXX
  8343 %}
  8345 // Conditional-store of the updated heap-top.
  8346 // Used during allocation of the shared heap.
  8347 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
  8349 instruct storePConditional(memory heap_top_ptr,
  8350                            rax_RegP oldval, rRegP newval,
  8351                            rFlagsReg cr)
  8352 %{
  8353   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
  8355   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
  8356             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
  8357   opcode(0x0F, 0xB1);
  8358   ins_encode(lock_prefix,
  8359              REX_reg_mem_wide(newval, heap_top_ptr),
  8360              OpcP, OpcS,
  8361              reg_mem(newval, heap_top_ptr));
  8362   ins_pipe(pipe_cmpxchg);
  8363 %}
  8365 // Conditional-store of an int value.
  8366 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
  8367 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
  8368 %{
  8369   match(Set cr (StoreIConditional mem (Binary oldval newval)));
  8370   effect(KILL oldval);
  8372   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
  8373   opcode(0x0F, 0xB1);
  8374   ins_encode(lock_prefix,
  8375              REX_reg_mem(newval, mem),
  8376              OpcP, OpcS,
  8377              reg_mem(newval, mem));
  8378   ins_pipe(pipe_cmpxchg);
  8379 %}
  8381 // Conditional-store of a long value.
  8382 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
  8383 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
  8384 %{
  8385   match(Set cr (StoreLConditional mem (Binary oldval newval)));
  8386   effect(KILL oldval);
  8388   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
  8389   opcode(0x0F, 0xB1);
  8390   ins_encode(lock_prefix,
  8391              REX_reg_mem_wide(newval, mem),
  8392              OpcP, OpcS,
  8393              reg_mem(newval, mem));
  8394   ins_pipe(pipe_cmpxchg);
  8395 %}
  8398 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
  8399 instruct compareAndSwapP(rRegI res,
  8400                          memory mem_ptr,
  8401                          rax_RegP oldval, rRegP newval,
  8402                          rFlagsReg cr)
  8403 %{
  8404   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
  8405   effect(KILL cr, KILL oldval);
  8407   format %{ "cmpxchgq $mem_ptr,$newval\t# "
  8408             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
  8409             "sete    $res\n\t"
  8410             "movzbl  $res, $res" %}
  8411   opcode(0x0F, 0xB1);
  8412   ins_encode(lock_prefix,
  8413              REX_reg_mem_wide(newval, mem_ptr),
  8414              OpcP, OpcS,
  8415              reg_mem(newval, mem_ptr),
  8416              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
  8417              REX_reg_breg(res, res), // movzbl
  8418              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
  8419   ins_pipe( pipe_cmpxchg );
  8420 %}
  8422 instruct compareAndSwapL(rRegI res,
  8423                          memory mem_ptr,
  8424                          rax_RegL oldval, rRegL newval,
  8425                          rFlagsReg cr)
  8426 %{
  8427   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
  8428   effect(KILL cr, KILL oldval);
  8430   format %{ "cmpxchgq $mem_ptr,$newval\t# "
  8431             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
  8432             "sete    $res\n\t"
  8433             "movzbl  $res, $res" %}
  8434   opcode(0x0F, 0xB1);
  8435   ins_encode(lock_prefix,
  8436              REX_reg_mem_wide(newval, mem_ptr),
  8437              OpcP, OpcS,
  8438              reg_mem(newval, mem_ptr),
  8439              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
  8440              REX_reg_breg(res, res), // movzbl
  8441              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
  8442   ins_pipe( pipe_cmpxchg );
  8443 %}
  8445 instruct compareAndSwapI(rRegI res,
  8446                          memory mem_ptr,
  8447                          rax_RegI oldval, rRegI newval,
  8448                          rFlagsReg cr)
  8449 %{
  8450   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
  8451   effect(KILL cr, KILL oldval);
  8453   format %{ "cmpxchgl $mem_ptr,$newval\t# "
  8454             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
  8455             "sete    $res\n\t"
  8456             "movzbl  $res, $res" %}
  8457   opcode(0x0F, 0xB1);
  8458   ins_encode(lock_prefix,
  8459              REX_reg_mem(newval, mem_ptr),
  8460              OpcP, OpcS,
  8461              reg_mem(newval, mem_ptr),
  8462              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
  8463              REX_reg_breg(res, res), // movzbl
  8464              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
  8465   ins_pipe( pipe_cmpxchg );
  8466 %}
  8469 instruct compareAndSwapN(rRegI res,
  8470                           memory mem_ptr,
  8471                           rax_RegN oldval, rRegN newval,
  8472                           rFlagsReg cr) %{
  8473   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
  8474   effect(KILL cr, KILL oldval);
  8476   format %{ "cmpxchgl $mem_ptr,$newval\t# "
  8477             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
  8478             "sete    $res\n\t"
  8479             "movzbl  $res, $res" %}
  8480   opcode(0x0F, 0xB1);
  8481   ins_encode(lock_prefix,
  8482              REX_reg_mem(newval, mem_ptr),
  8483              OpcP, OpcS,
  8484              reg_mem(newval, mem_ptr),
  8485              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
  8486              REX_reg_breg(res, res), // movzbl
  8487              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
  8488   ins_pipe( pipe_cmpxchg );
  8489 %}
  8491 //----------Subtraction Instructions-------------------------------------------
  8493 // Integer Subtraction Instructions
  8494 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
  8495 %{
  8496   match(Set dst (SubI dst src));
  8497   effect(KILL cr);
  8499   format %{ "subl    $dst, $src\t# int" %}
  8500   opcode(0x2B);
  8501   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
  8502   ins_pipe(ialu_reg_reg);
  8503 %}
  8505 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
  8506 %{
  8507   match(Set dst (SubI dst src));
  8508   effect(KILL cr);
  8510   format %{ "subl    $dst, $src\t# int" %}
  8511   opcode(0x81, 0x05);  /* Opcode 81 /5 */
  8512   ins_encode(OpcSErm(dst, src), Con8or32(src));
  8513   ins_pipe(ialu_reg);
  8514 %}
  8516 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
  8517 %{
  8518   match(Set dst (SubI dst (LoadI src)));
  8519   effect(KILL cr);
  8521   ins_cost(125);
  8522   format %{ "subl    $dst, $src\t# int" %}
  8523   opcode(0x2B);
  8524   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
  8525   ins_pipe(ialu_reg_mem);
  8526 %}
  8528 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
  8529 %{
  8530   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
  8531   effect(KILL cr);
  8533   ins_cost(150);
  8534   format %{ "subl    $dst, $src\t# int" %}
  8535   opcode(0x29); /* Opcode 29 /r */
  8536   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
  8537   ins_pipe(ialu_mem_reg);
  8538 %}
  8540 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
  8541 %{
  8542   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
  8543   effect(KILL cr);
  8545   ins_cost(125); // XXX
  8546   format %{ "subl    $dst, $src\t# int" %}
  8547   opcode(0x81); /* Opcode 81 /5 id */
  8548   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
  8549   ins_pipe(ialu_mem_imm);
  8550 %}
  8552 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
  8553 %{
  8554   match(Set dst (SubL dst src));
  8555   effect(KILL cr);
  8557   format %{ "subq    $dst, $src\t# long" %}
  8558   opcode(0x2B);
  8559   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
  8560   ins_pipe(ialu_reg_reg);
  8561 %}
  8563 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
  8564 %{
  8565   match(Set dst (SubL dst src));
  8566   effect(KILL cr);
  8568   format %{ "subq    $dst, $src\t# long" %}
  8569   opcode(0x81, 0x05);  /* Opcode 81 /5 */
  8570   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
  8571   ins_pipe(ialu_reg);
  8572 %}
  8574 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
  8575 %{
  8576   match(Set dst (SubL dst (LoadL src)));
  8577   effect(KILL cr);
  8579   ins_cost(125);
  8580   format %{ "subq    $dst, $src\t# long" %}
  8581   opcode(0x2B);
  8582   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
  8583   ins_pipe(ialu_reg_mem);
  8584 %}
  8586 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
  8587 %{
  8588   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
  8589   effect(KILL cr);
  8591   ins_cost(150);
  8592   format %{ "subq    $dst, $src\t# long" %}
  8593   opcode(0x29); /* Opcode 29 /r */
  8594   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
  8595   ins_pipe(ialu_mem_reg);
  8596 %}
  8598 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
  8599 %{
  8600   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
  8601   effect(KILL cr);
  8603   ins_cost(125); // XXX
  8604   format %{ "subq    $dst, $src\t# long" %}
  8605   opcode(0x81); /* Opcode 81 /5 id */
  8606   ins_encode(REX_mem_wide(dst),
  8607              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
  8608   ins_pipe(ialu_mem_imm);
  8609 %}
  8611 // Subtract from a pointer
  8612 // XXX hmpf???
  8613 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
  8614 %{
  8615   match(Set dst (AddP dst (SubI zero src)));
  8616   effect(KILL cr);
  8618   format %{ "subq    $dst, $src\t# ptr - int" %}
  8619   opcode(0x2B);
  8620   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
  8621   ins_pipe(ialu_reg_reg);
  8622 %}
  8624 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
  8625 %{
  8626   match(Set dst (SubI zero dst));
  8627   effect(KILL cr);
  8629   format %{ "negl    $dst\t# int" %}
  8630   opcode(0xF7, 0x03);  // Opcode F7 /3
  8631   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  8632   ins_pipe(ialu_reg);
  8633 %}
  8635 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
  8636 %{
  8637   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
  8638   effect(KILL cr);
  8640   format %{ "negl    $dst\t# int" %}
  8641   opcode(0xF7, 0x03);  // Opcode F7 /3
  8642   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
  8643   ins_pipe(ialu_reg);
  8644 %}
  8646 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
  8647 %{
  8648   match(Set dst (SubL zero dst));
  8649   effect(KILL cr);
  8651   format %{ "negq    $dst\t# long" %}
  8652   opcode(0xF7, 0x03);  // Opcode F7 /3
  8653   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  8654   ins_pipe(ialu_reg);
  8655 %}
  8657 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
  8658 %{
  8659   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
  8660   effect(KILL cr);
  8662   format %{ "negq    $dst\t# long" %}
  8663   opcode(0xF7, 0x03);  // Opcode F7 /3
  8664   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
  8665   ins_pipe(ialu_reg);
  8666 %}
  8669 //----------Multiplication/Division Instructions-------------------------------
  8670 // Integer Multiplication Instructions
  8671 // Multiply Register
  8673 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
  8674 %{
  8675   match(Set dst (MulI dst src));
  8676   effect(KILL cr);
  8678   ins_cost(300);
  8679   format %{ "imull   $dst, $src\t# int" %}
  8680   opcode(0x0F, 0xAF);
  8681   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
  8682   ins_pipe(ialu_reg_reg_alu0);
  8683 %}
  8685 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
  8686 %{
  8687   match(Set dst (MulI src imm));
  8688   effect(KILL cr);
  8690   ins_cost(300);
  8691   format %{ "imull   $dst, $src, $imm\t# int" %}
  8692   opcode(0x69); /* 69 /r id */
  8693   ins_encode(REX_reg_reg(dst, src),
  8694              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
  8695   ins_pipe(ialu_reg_reg_alu0);
  8696 %}
  8698 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
  8699 %{
  8700   match(Set dst (MulI dst (LoadI src)));
  8701   effect(KILL cr);
  8703   ins_cost(350);
  8704   format %{ "imull   $dst, $src\t# int" %}
  8705   opcode(0x0F, 0xAF);
  8706   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
  8707   ins_pipe(ialu_reg_mem_alu0);
  8708 %}
  8710 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
  8711 %{
  8712   match(Set dst (MulI (LoadI src) imm));
  8713   effect(KILL cr);
  8715   ins_cost(300);
  8716   format %{ "imull   $dst, $src, $imm\t# int" %}
  8717   opcode(0x69); /* 69 /r id */
  8718   ins_encode(REX_reg_mem(dst, src),
  8719              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
  8720   ins_pipe(ialu_reg_mem_alu0);
  8721 %}
  8723 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
  8724 %{
  8725   match(Set dst (MulL dst src));
  8726   effect(KILL cr);
  8728   ins_cost(300);
  8729   format %{ "imulq   $dst, $src\t# long" %}
  8730   opcode(0x0F, 0xAF);
  8731   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
  8732   ins_pipe(ialu_reg_reg_alu0);
  8733 %}
  8735 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
  8736 %{
  8737   match(Set dst (MulL src imm));
  8738   effect(KILL cr);
  8740   ins_cost(300);
  8741   format %{ "imulq   $dst, $src, $imm\t# long" %}
  8742   opcode(0x69); /* 69 /r id */
  8743   ins_encode(REX_reg_reg_wide(dst, src),
  8744              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
  8745   ins_pipe(ialu_reg_reg_alu0);
  8746 %}
  8748 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
  8749 %{
  8750   match(Set dst (MulL dst (LoadL src)));
  8751   effect(KILL cr);
  8753   ins_cost(350);
  8754   format %{ "imulq   $dst, $src\t# long" %}
  8755   opcode(0x0F, 0xAF);
  8756   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
  8757   ins_pipe(ialu_reg_mem_alu0);
  8758 %}
  8760 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
  8761 %{
  8762   match(Set dst (MulL (LoadL src) imm));
  8763   effect(KILL cr);
  8765   ins_cost(300);
  8766   format %{ "imulq   $dst, $src, $imm\t# long" %}
  8767   opcode(0x69); /* 69 /r id */
  8768   ins_encode(REX_reg_mem_wide(dst, src),
  8769              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
  8770   ins_pipe(ialu_reg_mem_alu0);
  8771 %}
  8773 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
  8774 %{
  8775   match(Set dst (MulHiL src rax));
  8776   effect(USE_KILL rax, KILL cr);
  8778   ins_cost(300);
  8779   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
  8780   opcode(0xF7, 0x5); /* Opcode F7 /5 */
  8781   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
  8782   ins_pipe(ialu_reg_reg_alu0);
  8783 %}
  8785 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
  8786                    rFlagsReg cr)
  8787 %{
  8788   match(Set rax (DivI rax div));
  8789   effect(KILL rdx, KILL cr);
  8791   ins_cost(30*100+10*100); // XXX
  8792   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
  8793             "jne,s   normal\n\t"
  8794             "xorl    rdx, rdx\n\t"
  8795             "cmpl    $div, -1\n\t"
  8796             "je,s    done\n"
  8797     "normal: cdql\n\t"
  8798             "idivl   $div\n"
  8799     "done:"        %}
  8800   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
  8801   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
  8802   ins_pipe(ialu_reg_reg_alu0);
  8803 %}
  8805 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
  8806                    rFlagsReg cr)
  8807 %{
  8808   match(Set rax (DivL rax div));
  8809   effect(KILL rdx, KILL cr);
  8811   ins_cost(30*100+10*100); // XXX
  8812   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
  8813             "cmpq    rax, rdx\n\t"
  8814             "jne,s   normal\n\t"
  8815             "xorl    rdx, rdx\n\t"
  8816             "cmpq    $div, -1\n\t"
  8817             "je,s    done\n"
  8818     "normal: cdqq\n\t"
  8819             "idivq   $div\n"
  8820     "done:"        %}
  8821   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
  8822   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
  8823   ins_pipe(ialu_reg_reg_alu0);
  8824 %}
  8826 // Integer DIVMOD with Register, both quotient and mod results
  8827 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
  8828                              rFlagsReg cr)
  8829 %{
  8830   match(DivModI rax div);
  8831   effect(KILL cr);
  8833   ins_cost(30*100+10*100); // XXX
  8834   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
  8835             "jne,s   normal\n\t"
  8836             "xorl    rdx, rdx\n\t"
  8837             "cmpl    $div, -1\n\t"
  8838             "je,s    done\n"
  8839     "normal: cdql\n\t"
  8840             "idivl   $div\n"
  8841     "done:"        %}
  8842   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
  8843   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
  8844   ins_pipe(pipe_slow);
  8845 %}
  8847 // Long DIVMOD with Register, both quotient and mod results
  8848 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
  8849                              rFlagsReg cr)
  8850 %{
  8851   match(DivModL rax div);
  8852   effect(KILL cr);
  8854   ins_cost(30*100+10*100); // XXX
  8855   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
  8856             "cmpq    rax, rdx\n\t"
  8857             "jne,s   normal\n\t"
  8858             "xorl    rdx, rdx\n\t"
  8859             "cmpq    $div, -1\n\t"
  8860             "je,s    done\n"
  8861     "normal: cdqq\n\t"
  8862             "idivq   $div\n"
  8863     "done:"        %}
  8864   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
  8865   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
  8866   ins_pipe(pipe_slow);
  8867 %}
  8869 //----------- DivL-By-Constant-Expansions--------------------------------------
  8870 // DivI cases are handled by the compiler
  8872 // Magic constant, reciprocal of 10
  8873 instruct loadConL_0x6666666666666667(rRegL dst)
  8874 %{
  8875   effect(DEF dst);
  8877   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
  8878   ins_encode(load_immL(dst, 0x6666666666666667));
  8879   ins_pipe(ialu_reg);
  8880 %}
  8882 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
  8883 %{
  8884   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
  8886   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
  8887   opcode(0xF7, 0x5); /* Opcode F7 /5 */
  8888   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
  8889   ins_pipe(ialu_reg_reg_alu0);
  8890 %}
  8892 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
  8893 %{
  8894   effect(USE_DEF dst, KILL cr);
  8896   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
  8897   opcode(0xC1, 0x7); /* C1 /7 ib */
  8898   ins_encode(reg_opc_imm_wide(dst, 0x3F));
  8899   ins_pipe(ialu_reg);
  8900 %}
  8902 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
  8903 %{
  8904   effect(USE_DEF dst, KILL cr);
  8906   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
  8907   opcode(0xC1, 0x7); /* C1 /7 ib */
  8908   ins_encode(reg_opc_imm_wide(dst, 0x2));
  8909   ins_pipe(ialu_reg);
  8910 %}
  8912 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
  8913 %{
  8914   match(Set dst (DivL src div));
  8916   ins_cost((5+8)*100);
  8917   expand %{
  8918     rax_RegL rax;                     // Killed temp
  8919     rFlagsReg cr;                     // Killed
  8920     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
  8921     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
  8922     sarL_rReg_63(src, cr);            // sarq  src, 63
  8923     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
  8924     subL_rReg(dst, src, cr);          // subl  rdx, src
  8925   %}
  8926 %}
  8928 //-----------------------------------------------------------------------------
  8930 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
  8931                    rFlagsReg cr)
  8932 %{
  8933   match(Set rdx (ModI rax div));
  8934   effect(KILL rax, KILL cr);
  8936   ins_cost(300); // XXX
  8937   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
  8938             "jne,s   normal\n\t"
  8939             "xorl    rdx, rdx\n\t"
  8940             "cmpl    $div, -1\n\t"
  8941             "je,s    done\n"
  8942     "normal: cdql\n\t"
  8943             "idivl   $div\n"
  8944     "done:"        %}
  8945   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
  8946   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
  8947   ins_pipe(ialu_reg_reg_alu0);
  8948 %}
  8950 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
  8951                    rFlagsReg cr)
  8952 %{
  8953   match(Set rdx (ModL rax div));
  8954   effect(KILL rax, KILL cr);
  8956   ins_cost(300); // XXX
  8957   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
  8958             "cmpq    rax, rdx\n\t"
  8959             "jne,s   normal\n\t"
  8960             "xorl    rdx, rdx\n\t"
  8961             "cmpq    $div, -1\n\t"
  8962             "je,s    done\n"
  8963     "normal: cdqq\n\t"
  8964             "idivq   $div\n"
  8965     "done:"        %}
  8966   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
  8967   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
  8968   ins_pipe(ialu_reg_reg_alu0);
  8969 %}
  8971 // Integer Shift Instructions
  8972 // Shift Left by one
  8973 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
  8974 %{
  8975   match(Set dst (LShiftI dst shift));
  8976   effect(KILL cr);
  8978   format %{ "sall    $dst, $shift" %}
  8979   opcode(0xD1, 0x4); /* D1 /4 */
  8980   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  8981   ins_pipe(ialu_reg);
  8982 %}
  8984 // Shift Left by one
  8985 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
  8986 %{
  8987   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
  8988   effect(KILL cr);
  8990   format %{ "sall    $dst, $shift\t" %}
  8991   opcode(0xD1, 0x4); /* D1 /4 */
  8992   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
  8993   ins_pipe(ialu_mem_imm);
  8994 %}
  8996 // Shift Left by 8-bit immediate
  8997 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
  8998 %{
  8999   match(Set dst (LShiftI dst shift));
  9000   effect(KILL cr);
  9002   format %{ "sall    $dst, $shift" %}
  9003   opcode(0xC1, 0x4); /* C1 /4 ib */
  9004   ins_encode(reg_opc_imm(dst, shift));
  9005   ins_pipe(ialu_reg);
  9006 %}
  9008 // Shift Left by 8-bit immediate
  9009 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
  9010 %{
  9011   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
  9012   effect(KILL cr);
  9014   format %{ "sall    $dst, $shift" %}
  9015   opcode(0xC1, 0x4); /* C1 /4 ib */
  9016   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
  9017   ins_pipe(ialu_mem_imm);
  9018 %}
  9020 // Shift Left by variable
  9021 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
  9022 %{
  9023   match(Set dst (LShiftI dst shift));
  9024   effect(KILL cr);
  9026   format %{ "sall    $dst, $shift" %}
  9027   opcode(0xD3, 0x4); /* D3 /4 */
  9028   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  9029   ins_pipe(ialu_reg_reg);
  9030 %}
  9032 // Shift Left by variable
  9033 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
  9034 %{
  9035   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
  9036   effect(KILL cr);
  9038   format %{ "sall    $dst, $shift" %}
  9039   opcode(0xD3, 0x4); /* D3 /4 */
  9040   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
  9041   ins_pipe(ialu_mem_reg);
  9042 %}
  9044 // Arithmetic shift right by one
  9045 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
  9046 %{
  9047   match(Set dst (RShiftI dst shift));
  9048   effect(KILL cr);
  9050   format %{ "sarl    $dst, $shift" %}
  9051   opcode(0xD1, 0x7); /* D1 /7 */
  9052   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  9053   ins_pipe(ialu_reg);
  9054 %}
  9056 // Arithmetic shift right by one
  9057 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
  9058 %{
  9059   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
  9060   effect(KILL cr);
  9062   format %{ "sarl    $dst, $shift" %}
  9063   opcode(0xD1, 0x7); /* D1 /7 */
  9064   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
  9065   ins_pipe(ialu_mem_imm);
  9066 %}
  9068 // Arithmetic Shift Right by 8-bit immediate
  9069 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
  9070 %{
  9071   match(Set dst (RShiftI dst shift));
  9072   effect(KILL cr);
  9074   format %{ "sarl    $dst, $shift" %}
  9075   opcode(0xC1, 0x7); /* C1 /7 ib */
  9076   ins_encode(reg_opc_imm(dst, shift));
  9077   ins_pipe(ialu_mem_imm);
  9078 %}
  9080 // Arithmetic Shift Right by 8-bit immediate
  9081 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
  9082 %{
  9083   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
  9084   effect(KILL cr);
  9086   format %{ "sarl    $dst, $shift" %}
  9087   opcode(0xC1, 0x7); /* C1 /7 ib */
  9088   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
  9089   ins_pipe(ialu_mem_imm);
  9090 %}
  9092 // Arithmetic Shift Right by variable
  9093 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
  9094 %{
  9095   match(Set dst (RShiftI dst shift));
  9096   effect(KILL cr);
  9098   format %{ "sarl    $dst, $shift" %}
  9099   opcode(0xD3, 0x7); /* D3 /7 */
  9100   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  9101   ins_pipe(ialu_reg_reg);
  9102 %}
  9104 // Arithmetic Shift Right by variable
  9105 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
  9106 %{
  9107   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
  9108   effect(KILL cr);
  9110   format %{ "sarl    $dst, $shift" %}
  9111   opcode(0xD3, 0x7); /* D3 /7 */
  9112   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
  9113   ins_pipe(ialu_mem_reg);
  9114 %}
  9116 // Logical shift right by one
  9117 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
  9118 %{
  9119   match(Set dst (URShiftI dst shift));
  9120   effect(KILL cr);
  9122   format %{ "shrl    $dst, $shift" %}
  9123   opcode(0xD1, 0x5); /* D1 /5 */
  9124   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  9125   ins_pipe(ialu_reg);
  9126 %}
  9128 // Logical shift right by one
  9129 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
  9130 %{
  9131   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
  9132   effect(KILL cr);
  9134   format %{ "shrl    $dst, $shift" %}
  9135   opcode(0xD1, 0x5); /* D1 /5 */
  9136   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
  9137   ins_pipe(ialu_mem_imm);
  9138 %}
  9140 // Logical Shift Right by 8-bit immediate
  9141 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
  9142 %{
  9143   match(Set dst (URShiftI dst shift));
  9144   effect(KILL cr);
  9146   format %{ "shrl    $dst, $shift" %}
  9147   opcode(0xC1, 0x5); /* C1 /5 ib */
  9148   ins_encode(reg_opc_imm(dst, shift));
  9149   ins_pipe(ialu_reg);
  9150 %}
  9152 // Logical Shift Right by 8-bit immediate
  9153 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
  9154 %{
  9155   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
  9156   effect(KILL cr);
  9158   format %{ "shrl    $dst, $shift" %}
  9159   opcode(0xC1, 0x5); /* C1 /5 ib */
  9160   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
  9161   ins_pipe(ialu_mem_imm);
  9162 %}
  9164 // Logical Shift Right by variable
  9165 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
  9166 %{
  9167   match(Set dst (URShiftI dst shift));
  9168   effect(KILL cr);
  9170   format %{ "shrl    $dst, $shift" %}
  9171   opcode(0xD3, 0x5); /* D3 /5 */
  9172   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  9173   ins_pipe(ialu_reg_reg);
  9174 %}
  9176 // Logical Shift Right by variable
  9177 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
  9178 %{
  9179   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
  9180   effect(KILL cr);
  9182   format %{ "shrl    $dst, $shift" %}
  9183   opcode(0xD3, 0x5); /* D3 /5 */
  9184   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
  9185   ins_pipe(ialu_mem_reg);
  9186 %}
  9188 // Long Shift Instructions
  9189 // Shift Left by one
  9190 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
  9191 %{
  9192   match(Set dst (LShiftL dst shift));
  9193   effect(KILL cr);
  9195   format %{ "salq    $dst, $shift" %}
  9196   opcode(0xD1, 0x4); /* D1 /4 */
  9197   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  9198   ins_pipe(ialu_reg);
  9199 %}
  9201 // Shift Left by one
  9202 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
  9203 %{
  9204   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
  9205   effect(KILL cr);
  9207   format %{ "salq    $dst, $shift" %}
  9208   opcode(0xD1, 0x4); /* D1 /4 */
  9209   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
  9210   ins_pipe(ialu_mem_imm);
  9211 %}
  9213 // Shift Left by 8-bit immediate
  9214 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
  9215 %{
  9216   match(Set dst (LShiftL dst shift));
  9217   effect(KILL cr);
  9219   format %{ "salq    $dst, $shift" %}
  9220   opcode(0xC1, 0x4); /* C1 /4 ib */
  9221   ins_encode(reg_opc_imm_wide(dst, shift));
  9222   ins_pipe(ialu_reg);
  9223 %}
  9225 // Shift Left by 8-bit immediate
  9226 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
  9227 %{
  9228   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
  9229   effect(KILL cr);
  9231   format %{ "salq    $dst, $shift" %}
  9232   opcode(0xC1, 0x4); /* C1 /4 ib */
  9233   ins_encode(REX_mem_wide(dst), OpcP,
  9234              RM_opc_mem(secondary, dst), Con8or32(shift));
  9235   ins_pipe(ialu_mem_imm);
  9236 %}
  9238 // Shift Left by variable
  9239 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
  9240 %{
  9241   match(Set dst (LShiftL dst shift));
  9242   effect(KILL cr);
  9244   format %{ "salq    $dst, $shift" %}
  9245   opcode(0xD3, 0x4); /* D3 /4 */
  9246   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  9247   ins_pipe(ialu_reg_reg);
  9248 %}
  9250 // Shift Left by variable
  9251 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
  9252 %{
  9253   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
  9254   effect(KILL cr);
  9256   format %{ "salq    $dst, $shift" %}
  9257   opcode(0xD3, 0x4); /* D3 /4 */
  9258   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
  9259   ins_pipe(ialu_mem_reg);
  9260 %}
  9262 // Arithmetic shift right by one
  9263 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
  9264 %{
  9265   match(Set dst (RShiftL dst shift));
  9266   effect(KILL cr);
  9268   format %{ "sarq    $dst, $shift" %}
  9269   opcode(0xD1, 0x7); /* D1 /7 */
  9270   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  9271   ins_pipe(ialu_reg);
  9272 %}
  9274 // Arithmetic shift right by one
  9275 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
  9276 %{
  9277   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
  9278   effect(KILL cr);
  9280   format %{ "sarq    $dst, $shift" %}
  9281   opcode(0xD1, 0x7); /* D1 /7 */
  9282   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
  9283   ins_pipe(ialu_mem_imm);
  9284 %}
  9286 // Arithmetic Shift Right by 8-bit immediate
  9287 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
  9288 %{
  9289   match(Set dst (RShiftL dst shift));
  9290   effect(KILL cr);
  9292   format %{ "sarq    $dst, $shift" %}
  9293   opcode(0xC1, 0x7); /* C1 /7 ib */
  9294   ins_encode(reg_opc_imm_wide(dst, shift));
  9295   ins_pipe(ialu_mem_imm);
  9296 %}
  9298 // Arithmetic Shift Right by 8-bit immediate
  9299 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
  9300 %{
  9301   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
  9302   effect(KILL cr);
  9304   format %{ "sarq    $dst, $shift" %}
  9305   opcode(0xC1, 0x7); /* C1 /7 ib */
  9306   ins_encode(REX_mem_wide(dst), OpcP,
  9307              RM_opc_mem(secondary, dst), Con8or32(shift));
  9308   ins_pipe(ialu_mem_imm);
  9309 %}
  9311 // Arithmetic Shift Right by variable
  9312 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
  9313 %{
  9314   match(Set dst (RShiftL dst shift));
  9315   effect(KILL cr);
  9317   format %{ "sarq    $dst, $shift" %}
  9318   opcode(0xD3, 0x7); /* D3 /7 */
  9319   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  9320   ins_pipe(ialu_reg_reg);
  9321 %}
  9323 // Arithmetic Shift Right by variable
  9324 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
  9325 %{
  9326   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
  9327   effect(KILL cr);
  9329   format %{ "sarq    $dst, $shift" %}
  9330   opcode(0xD3, 0x7); /* D3 /7 */
  9331   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
  9332   ins_pipe(ialu_mem_reg);
  9333 %}
  9335 // Logical shift right by one
  9336 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
  9337 %{
  9338   match(Set dst (URShiftL dst shift));
  9339   effect(KILL cr);
  9341   format %{ "shrq    $dst, $shift" %}
  9342   opcode(0xD1, 0x5); /* D1 /5 */
  9343   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
  9344   ins_pipe(ialu_reg);
  9345 %}
  9347 // Logical shift right by one
  9348 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
  9349 %{
  9350   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
  9351   effect(KILL cr);
  9353   format %{ "shrq    $dst, $shift" %}
  9354   opcode(0xD1, 0x5); /* D1 /5 */
  9355   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
  9356   ins_pipe(ialu_mem_imm);
  9357 %}
  9359 // Logical Shift Right by 8-bit immediate
  9360 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
  9361 %{
  9362   match(Set dst (URShiftL dst shift));
  9363   effect(KILL cr);
  9365   format %{ "shrq    $dst, $shift" %}
  9366   opcode(0xC1, 0x5); /* C1 /5 ib */
  9367   ins_encode(reg_opc_imm_wide(dst, shift));
  9368   ins_pipe(ialu_reg);
  9369 %}
  9372 // Logical Shift Right by 8-bit immediate
  9373 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
  9374 %{
  9375   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
  9376   effect(KILL cr);
  9378   format %{ "shrq    $dst, $shift" %}
  9379   opcode(0xC1, 0x5); /* C1 /5 ib */
  9380   ins_encode(REX_mem_wide(dst), OpcP,
  9381              RM_opc_mem(secondary, dst), Con8or32(shift));
  9382   ins_pipe(ialu_mem_imm);
  9383 %}
  9385 // Logical Shift Right by variable
  9386 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
  9387 %{
  9388   match(Set dst (URShiftL dst shift));
  9389   effect(KILL cr);
  9391   format %{ "shrq    $dst, $shift" %}
  9392   opcode(0xD3, 0x5); /* D3 /5 */
  9393   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  9394   ins_pipe(ialu_reg_reg);
  9395 %}
  9397 // Logical Shift Right by variable
  9398 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
  9399 %{
  9400   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
  9401   effect(KILL cr);
  9403   format %{ "shrq    $dst, $shift" %}
  9404   opcode(0xD3, 0x5); /* D3 /5 */
  9405   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
  9406   ins_pipe(ialu_mem_reg);
  9407 %}
  9409 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
  9410 // This idiom is used by the compiler for the i2b bytecode.
  9411 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
  9412 %{
  9413   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
  9415   format %{ "movsbl  $dst, $src\t# i2b" %}
  9416   opcode(0x0F, 0xBE);
  9417   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
  9418   ins_pipe(ialu_reg_reg);
  9419 %}
  9421 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
  9422 // This idiom is used by the compiler the i2s bytecode.
  9423 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
  9424 %{
  9425   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
  9427   format %{ "movswl  $dst, $src\t# i2s" %}
  9428   opcode(0x0F, 0xBF);
  9429   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
  9430   ins_pipe(ialu_reg_reg);
  9431 %}
  9433 // ROL/ROR instructions
  9435 // ROL expand
  9436 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
  9437   effect(KILL cr, USE_DEF dst);
  9439   format %{ "roll    $dst" %}
  9440   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
  9441   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  9442   ins_pipe(ialu_reg);
  9443 %}
  9445 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
  9446   effect(USE_DEF dst, USE shift, KILL cr);
  9448   format %{ "roll    $dst, $shift" %}
  9449   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
  9450   ins_encode( reg_opc_imm(dst, shift) );
  9451   ins_pipe(ialu_reg);
  9452 %}
  9454 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
  9455 %{
  9456   effect(USE_DEF dst, USE shift, KILL cr);
  9458   format %{ "roll    $dst, $shift" %}
  9459   opcode(0xD3, 0x0); /* Opcode D3 /0 */
  9460   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  9461   ins_pipe(ialu_reg_reg);
  9462 %}
  9463 // end of ROL expand
  9465 // Rotate Left by one
  9466 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
  9467 %{
  9468   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
  9470   expand %{
  9471     rolI_rReg_imm1(dst, cr);
  9472   %}
  9473 %}
  9475 // Rotate Left by 8-bit immediate
  9476 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
  9477 %{
  9478   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
  9479   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
  9481   expand %{
  9482     rolI_rReg_imm8(dst, lshift, cr);
  9483   %}
  9484 %}
  9486 // Rotate Left by variable
  9487 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
  9488 %{
  9489   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
  9491   expand %{
  9492     rolI_rReg_CL(dst, shift, cr);
  9493   %}
  9494 %}
  9496 // Rotate Left by variable
  9497 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
  9498 %{
  9499   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
  9501   expand %{
  9502     rolI_rReg_CL(dst, shift, cr);
  9503   %}
  9504 %}
  9506 // ROR expand
  9507 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
  9508 %{
  9509   effect(USE_DEF dst, KILL cr);
  9511   format %{ "rorl    $dst" %}
  9512   opcode(0xD1, 0x1); /* D1 /1 */
  9513   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  9514   ins_pipe(ialu_reg);
  9515 %}
  9517 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
  9518 %{
  9519   effect(USE_DEF dst, USE shift, KILL cr);
  9521   format %{ "rorl    $dst, $shift" %}
  9522   opcode(0xC1, 0x1); /* C1 /1 ib */
  9523   ins_encode(reg_opc_imm(dst, shift));
  9524   ins_pipe(ialu_reg);
  9525 %}
  9527 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
  9528 %{
  9529   effect(USE_DEF dst, USE shift, KILL cr);
  9531   format %{ "rorl    $dst, $shift" %}
  9532   opcode(0xD3, 0x1); /* D3 /1 */
  9533   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
  9534   ins_pipe(ialu_reg_reg);
  9535 %}
  9536 // end of ROR expand
  9538 // Rotate Right by one
  9539 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
  9540 %{
  9541   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
  9543   expand %{
  9544     rorI_rReg_imm1(dst, cr);
  9545   %}
  9546 %}
  9548 // Rotate Right by 8-bit immediate
  9549 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
  9550 %{
  9551   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
  9552   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
  9554   expand %{
  9555     rorI_rReg_imm8(dst, rshift, cr);
  9556   %}
  9557 %}
  9559 // Rotate Right by variable
  9560 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
  9561 %{
  9562   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
  9564   expand %{
  9565     rorI_rReg_CL(dst, shift, cr);
  9566   %}
  9567 %}
  9569 // Rotate Right by variable
  9570 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
  9571 %{
  9572   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
  9574   expand %{
  9575     rorI_rReg_CL(dst, shift, cr);
  9576   %}
  9577 %}
  9579 // for long rotate
  9580 // ROL expand
  9581 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
  9582   effect(USE_DEF dst, KILL cr);
  9584   format %{ "rolq    $dst" %}
  9585   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
  9586   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  9587   ins_pipe(ialu_reg);
  9588 %}
  9590 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
  9591   effect(USE_DEF dst, USE shift, KILL cr);
  9593   format %{ "rolq    $dst, $shift" %}
  9594   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
  9595   ins_encode( reg_opc_imm_wide(dst, shift) );
  9596   ins_pipe(ialu_reg);
  9597 %}
  9599 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
  9600 %{
  9601   effect(USE_DEF dst, USE shift, KILL cr);
  9603   format %{ "rolq    $dst, $shift" %}
  9604   opcode(0xD3, 0x0); /* Opcode D3 /0 */
  9605   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  9606   ins_pipe(ialu_reg_reg);
  9607 %}
  9608 // end of ROL expand
  9610 // Rotate Left by one
  9611 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
  9612 %{
  9613   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
  9615   expand %{
  9616     rolL_rReg_imm1(dst, cr);
  9617   %}
  9618 %}
  9620 // Rotate Left by 8-bit immediate
  9621 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
  9622 %{
  9623   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
  9624   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
  9626   expand %{
  9627     rolL_rReg_imm8(dst, lshift, cr);
  9628   %}
  9629 %}
  9631 // Rotate Left by variable
  9632 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
  9633 %{
  9634   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
  9636   expand %{
  9637     rolL_rReg_CL(dst, shift, cr);
  9638   %}
  9639 %}
  9641 // Rotate Left by variable
  9642 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
  9643 %{
  9644   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
  9646   expand %{
  9647     rolL_rReg_CL(dst, shift, cr);
  9648   %}
  9649 %}
  9651 // ROR expand
  9652 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
  9653 %{
  9654   effect(USE_DEF dst, KILL cr);
  9656   format %{ "rorq    $dst" %}
  9657   opcode(0xD1, 0x1); /* D1 /1 */
  9658   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  9659   ins_pipe(ialu_reg);
  9660 %}
  9662 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
  9663 %{
  9664   effect(USE_DEF dst, USE shift, KILL cr);
  9666   format %{ "rorq    $dst, $shift" %}
  9667   opcode(0xC1, 0x1); /* C1 /1 ib */
  9668   ins_encode(reg_opc_imm_wide(dst, shift));
  9669   ins_pipe(ialu_reg);
  9670 %}
  9672 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
  9673 %{
  9674   effect(USE_DEF dst, USE shift, KILL cr);
  9676   format %{ "rorq    $dst, $shift" %}
  9677   opcode(0xD3, 0x1); /* D3 /1 */
  9678   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
  9679   ins_pipe(ialu_reg_reg);
  9680 %}
  9681 // end of ROR expand
  9683 // Rotate Right by one
  9684 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
  9685 %{
  9686   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
  9688   expand %{
  9689     rorL_rReg_imm1(dst, cr);
  9690   %}
  9691 %}
  9693 // Rotate Right by 8-bit immediate
  9694 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
  9695 %{
  9696   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
  9697   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
  9699   expand %{
  9700     rorL_rReg_imm8(dst, rshift, cr);
  9701   %}
  9702 %}
  9704 // Rotate Right by variable
  9705 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
  9706 %{
  9707   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
  9709   expand %{
  9710     rorL_rReg_CL(dst, shift, cr);
  9711   %}
  9712 %}
  9714 // Rotate Right by variable
  9715 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
  9716 %{
  9717   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
  9719   expand %{
  9720     rorL_rReg_CL(dst, shift, cr);
  9721   %}
  9722 %}
  9724 // Logical Instructions
  9726 // Integer Logical Instructions
  9728 // And Instructions
  9729 // And Register with Register
  9730 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
  9731 %{
  9732   match(Set dst (AndI dst src));
  9733   effect(KILL cr);
  9735   format %{ "andl    $dst, $src\t# int" %}
  9736   opcode(0x23);
  9737   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
  9738   ins_pipe(ialu_reg_reg);
  9739 %}
  9741 // And Register with Immediate 255
  9742 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
  9743 %{
  9744   match(Set dst (AndI dst src));
  9746   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
  9747   opcode(0x0F, 0xB6);
  9748   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
  9749   ins_pipe(ialu_reg);
  9750 %}
  9752 // And Register with Immediate 255 and promote to long
  9753 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
  9754 %{
  9755   match(Set dst (ConvI2L (AndI src mask)));
  9757   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
  9758   opcode(0x0F, 0xB6);
  9759   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
  9760   ins_pipe(ialu_reg);
  9761 %}
  9763 // And Register with Immediate 65535
  9764 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
  9765 %{
  9766   match(Set dst (AndI dst src));
  9768   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
  9769   opcode(0x0F, 0xB7);
  9770   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
  9771   ins_pipe(ialu_reg);
  9772 %}
  9774 // And Register with Immediate 65535 and promote to long
  9775 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
  9776 %{
  9777   match(Set dst (ConvI2L (AndI src mask)));
  9779   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
  9780   opcode(0x0F, 0xB7);
  9781   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
  9782   ins_pipe(ialu_reg);
  9783 %}
  9785 // And Register with Immediate
  9786 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
  9787 %{
  9788   match(Set dst (AndI dst src));
  9789   effect(KILL cr);
  9791   format %{ "andl    $dst, $src\t# int" %}
  9792   opcode(0x81, 0x04); /* Opcode 81 /4 */
  9793   ins_encode(OpcSErm(dst, src), Con8or32(src));
  9794   ins_pipe(ialu_reg);
  9795 %}
  9797 // And Register with Memory
  9798 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
  9799 %{
  9800   match(Set dst (AndI dst (LoadI src)));
  9801   effect(KILL cr);
  9803   ins_cost(125);
  9804   format %{ "andl    $dst, $src\t# int" %}
  9805   opcode(0x23);
  9806   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
  9807   ins_pipe(ialu_reg_mem);
  9808 %}
  9810 // And Memory with Register
  9811 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
  9812 %{
  9813   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
  9814   effect(KILL cr);
  9816   ins_cost(150);
  9817   format %{ "andl    $dst, $src\t# int" %}
  9818   opcode(0x21); /* Opcode 21 /r */
  9819   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
  9820   ins_pipe(ialu_mem_reg);
  9821 %}
  9823 // And Memory with Immediate
  9824 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
  9825 %{
  9826   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
  9827   effect(KILL cr);
  9829   ins_cost(125);
  9830   format %{ "andl    $dst, $src\t# int" %}
  9831   opcode(0x81, 0x4); /* Opcode 81 /4 id */
  9832   ins_encode(REX_mem(dst), OpcSE(src),
  9833              RM_opc_mem(secondary, dst), Con8or32(src));
  9834   ins_pipe(ialu_mem_imm);
  9835 %}
  9837 // Or Instructions
  9838 // Or Register with Register
  9839 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
  9840 %{
  9841   match(Set dst (OrI dst src));
  9842   effect(KILL cr);
  9844   format %{ "orl     $dst, $src\t# int" %}
  9845   opcode(0x0B);
  9846   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
  9847   ins_pipe(ialu_reg_reg);
  9848 %}
  9850 // Or Register with Immediate
  9851 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
  9852 %{
  9853   match(Set dst (OrI dst src));
  9854   effect(KILL cr);
  9856   format %{ "orl     $dst, $src\t# int" %}
  9857   opcode(0x81, 0x01); /* Opcode 81 /1 id */
  9858   ins_encode(OpcSErm(dst, src), Con8or32(src));
  9859   ins_pipe(ialu_reg);
  9860 %}
  9862 // Or Register with Memory
  9863 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
  9864 %{
  9865   match(Set dst (OrI dst (LoadI src)));
  9866   effect(KILL cr);
  9868   ins_cost(125);
  9869   format %{ "orl     $dst, $src\t# int" %}
  9870   opcode(0x0B);
  9871   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
  9872   ins_pipe(ialu_reg_mem);
  9873 %}
  9875 // Or Memory with Register
  9876 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
  9877 %{
  9878   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
  9879   effect(KILL cr);
  9881   ins_cost(150);
  9882   format %{ "orl     $dst, $src\t# int" %}
  9883   opcode(0x09); /* Opcode 09 /r */
  9884   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
  9885   ins_pipe(ialu_mem_reg);
  9886 %}
  9888 // Or Memory with Immediate
  9889 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
  9890 %{
  9891   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
  9892   effect(KILL cr);
  9894   ins_cost(125);
  9895   format %{ "orl     $dst, $src\t# int" %}
  9896   opcode(0x81, 0x1); /* Opcode 81 /1 id */
  9897   ins_encode(REX_mem(dst), OpcSE(src),
  9898              RM_opc_mem(secondary, dst), Con8or32(src));
  9899   ins_pipe(ialu_mem_imm);
  9900 %}
  9902 // Xor Instructions
  9903 // Xor Register with Register
  9904 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
  9905 %{
  9906   match(Set dst (XorI dst src));
  9907   effect(KILL cr);
  9909   format %{ "xorl    $dst, $src\t# int" %}
  9910   opcode(0x33);
  9911   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
  9912   ins_pipe(ialu_reg_reg);
  9913 %}
  9915 // Xor Register with Immediate -1
  9916 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
  9917   match(Set dst (XorI dst imm));  
  9919   format %{ "not    $dst" %}  
  9920   ins_encode %{
  9921      __ notl($dst$$Register);
  9922   %}
  9923   ins_pipe(ialu_reg);
  9924 %}
  9926 // Xor Register with Immediate
  9927 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
  9928 %{
  9929   match(Set dst (XorI dst src));
  9930   effect(KILL cr);
  9932   format %{ "xorl    $dst, $src\t# int" %}
  9933   opcode(0x81, 0x06); /* Opcode 81 /6 id */
  9934   ins_encode(OpcSErm(dst, src), Con8or32(src));
  9935   ins_pipe(ialu_reg);
  9936 %}
  9938 // Xor Register with Memory
  9939 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
  9940 %{
  9941   match(Set dst (XorI dst (LoadI src)));
  9942   effect(KILL cr);
  9944   ins_cost(125);
  9945   format %{ "xorl    $dst, $src\t# int" %}
  9946   opcode(0x33);
  9947   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
  9948   ins_pipe(ialu_reg_mem);
  9949 %}
  9951 // Xor Memory with Register
  9952 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
  9953 %{
  9954   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
  9955   effect(KILL cr);
  9957   ins_cost(150);
  9958   format %{ "xorl    $dst, $src\t# int" %}
  9959   opcode(0x31); /* Opcode 31 /r */
  9960   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
  9961   ins_pipe(ialu_mem_reg);
  9962 %}
  9964 // Xor Memory with Immediate
  9965 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
  9966 %{
  9967   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
  9968   effect(KILL cr);
  9970   ins_cost(125);
  9971   format %{ "xorl    $dst, $src\t# int" %}
  9972   opcode(0x81, 0x6); /* Opcode 81 /6 id */
  9973   ins_encode(REX_mem(dst), OpcSE(src),
  9974              RM_opc_mem(secondary, dst), Con8or32(src));
  9975   ins_pipe(ialu_mem_imm);
  9976 %}
  9979 // Long Logical Instructions
  9981 // And Instructions
  9982 // And Register with Register
  9983 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
  9984 %{
  9985   match(Set dst (AndL dst src));
  9986   effect(KILL cr);
  9988   format %{ "andq    $dst, $src\t# long" %}
  9989   opcode(0x23);
  9990   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
  9991   ins_pipe(ialu_reg_reg);
  9992 %}
  9994 // And Register with Immediate 255
  9995 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
  9996 %{
  9997   match(Set dst (AndL dst src));
  9999   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
 10000   opcode(0x0F, 0xB6);
 10001   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
 10002   ins_pipe(ialu_reg);
 10003 %}
 10005 // And Register with Immediate 65535
 10006 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
 10007 %{
 10008   match(Set dst (AndL dst src));
 10010   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
 10011   opcode(0x0F, 0xB7);
 10012   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
 10013   ins_pipe(ialu_reg);
 10014 %}
 10016 // And Register with Immediate
 10017 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 10018 %{
 10019   match(Set dst (AndL dst src));
 10020   effect(KILL cr);
 10022   format %{ "andq    $dst, $src\t# long" %}
 10023   opcode(0x81, 0x04); /* Opcode 81 /4 */
 10024   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
 10025   ins_pipe(ialu_reg);
 10026 %}
 10028 // And Register with Memory
 10029 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 10030 %{
 10031   match(Set dst (AndL dst (LoadL src)));
 10032   effect(KILL cr);
 10034   ins_cost(125);
 10035   format %{ "andq    $dst, $src\t# long" %}
 10036   opcode(0x23);
 10037   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 10038   ins_pipe(ialu_reg_mem);
 10039 %}
 10041 // And Memory with Register
 10042 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 10043 %{
 10044   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 10045   effect(KILL cr);
 10047   ins_cost(150);
 10048   format %{ "andq    $dst, $src\t# long" %}
 10049   opcode(0x21); /* Opcode 21 /r */
 10050   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 10051   ins_pipe(ialu_mem_reg);
 10052 %}
 10054 // And Memory with Immediate
 10055 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 10056 %{
 10057   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 10058   effect(KILL cr);
 10060   ins_cost(125);
 10061   format %{ "andq    $dst, $src\t# long" %}
 10062   opcode(0x81, 0x4); /* Opcode 81 /4 id */
 10063   ins_encode(REX_mem_wide(dst), OpcSE(src),
 10064              RM_opc_mem(secondary, dst), Con8or32(src));
 10065   ins_pipe(ialu_mem_imm);
 10066 %}
 10068 // Or Instructions
 10069 // Or Register with Register
 10070 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 10071 %{
 10072   match(Set dst (OrL dst src));
 10073   effect(KILL cr);
 10075   format %{ "orq     $dst, $src\t# long" %}
 10076   opcode(0x0B);
 10077   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 10078   ins_pipe(ialu_reg_reg);
 10079 %}
 10081 // Use any_RegP to match R15 (TLS register) without spilling.
 10082 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
 10083   match(Set dst (OrL dst (CastP2X src)));
 10084   effect(KILL cr);
 10086   format %{ "orq     $dst, $src\t# long" %}
 10087   opcode(0x0B);
 10088   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 10089   ins_pipe(ialu_reg_reg);
 10090 %}
 10093 // Or Register with Immediate
 10094 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 10095 %{
 10096   match(Set dst (OrL dst src));
 10097   effect(KILL cr);
 10099   format %{ "orq     $dst, $src\t# long" %}
 10100   opcode(0x81, 0x01); /* Opcode 81 /1 id */
 10101   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
 10102   ins_pipe(ialu_reg);
 10103 %}
 10105 // Or Register with Memory
 10106 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 10107 %{
 10108   match(Set dst (OrL dst (LoadL src)));
 10109   effect(KILL cr);
 10111   ins_cost(125);
 10112   format %{ "orq     $dst, $src\t# long" %}
 10113   opcode(0x0B);
 10114   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 10115   ins_pipe(ialu_reg_mem);
 10116 %}
 10118 // Or Memory with Register
 10119 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 10120 %{
 10121   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 10122   effect(KILL cr);
 10124   ins_cost(150);
 10125   format %{ "orq     $dst, $src\t# long" %}
 10126   opcode(0x09); /* Opcode 09 /r */
 10127   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 10128   ins_pipe(ialu_mem_reg);
 10129 %}
 10131 // Or Memory with Immediate
 10132 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 10133 %{
 10134   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 10135   effect(KILL cr);
 10137   ins_cost(125);
 10138   format %{ "orq     $dst, $src\t# long" %}
 10139   opcode(0x81, 0x1); /* Opcode 81 /1 id */
 10140   ins_encode(REX_mem_wide(dst), OpcSE(src),
 10141              RM_opc_mem(secondary, dst), Con8or32(src));
 10142   ins_pipe(ialu_mem_imm);
 10143 %}
 10145 // Xor Instructions
 10146 // Xor Register with Register
 10147 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 10148 %{
 10149   match(Set dst (XorL dst src));
 10150   effect(KILL cr);
 10152   format %{ "xorq    $dst, $src\t# long" %}
 10153   opcode(0x33);
 10154   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 10155   ins_pipe(ialu_reg_reg);
 10156 %}
 10158 // Xor Register with Immediate -1
 10159 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
 10160   match(Set dst (XorL dst imm));  
 10162   format %{ "notq   $dst" %}  
 10163   ins_encode %{
 10164      __ notq($dst$$Register);
 10165   %}
 10166   ins_pipe(ialu_reg);
 10167 %}
 10169 // Xor Register with Immediate
 10170 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 10171 %{
 10172   match(Set dst (XorL dst src));
 10173   effect(KILL cr);
 10175   format %{ "xorq    $dst, $src\t# long" %}
 10176   opcode(0x81, 0x06); /* Opcode 81 /6 id */
 10177   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
 10178   ins_pipe(ialu_reg);
 10179 %}
 10181 // Xor Register with Memory
 10182 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 10183 %{
 10184   match(Set dst (XorL dst (LoadL src)));
 10185   effect(KILL cr);
 10187   ins_cost(125);
 10188   format %{ "xorq    $dst, $src\t# long" %}
 10189   opcode(0x33);
 10190   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 10191   ins_pipe(ialu_reg_mem);
 10192 %}
 10194 // Xor Memory with Register
 10195 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 10196 %{
 10197   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 10198   effect(KILL cr);
 10200   ins_cost(150);
 10201   format %{ "xorq    $dst, $src\t# long" %}
 10202   opcode(0x31); /* Opcode 31 /r */
 10203   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 10204   ins_pipe(ialu_mem_reg);
 10205 %}
 10207 // Xor Memory with Immediate
 10208 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 10209 %{
 10210   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 10211   effect(KILL cr);
 10213   ins_cost(125);
 10214   format %{ "xorq    $dst, $src\t# long" %}
 10215   opcode(0x81, 0x6); /* Opcode 81 /6 id */
 10216   ins_encode(REX_mem_wide(dst), OpcSE(src),
 10217              RM_opc_mem(secondary, dst), Con8or32(src));
 10218   ins_pipe(ialu_mem_imm);
 10219 %}
 10221 // Convert Int to Boolean
 10222 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
 10223 %{
 10224   match(Set dst (Conv2B src));
 10225   effect(KILL cr);
 10227   format %{ "testl   $src, $src\t# ci2b\n\t"
 10228             "setnz   $dst\n\t"
 10229             "movzbl  $dst, $dst" %}
 10230   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
 10231              setNZ_reg(dst),
 10232              REX_reg_breg(dst, dst), // movzbl
 10233              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
 10234   ins_pipe(pipe_slow); // XXX
 10235 %}
 10237 // Convert Pointer to Boolean
 10238 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
 10239 %{
 10240   match(Set dst (Conv2B src));
 10241   effect(KILL cr);
 10243   format %{ "testq   $src, $src\t# cp2b\n\t"
 10244             "setnz   $dst\n\t"
 10245             "movzbl  $dst, $dst" %}
 10246   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
 10247              setNZ_reg(dst),
 10248              REX_reg_breg(dst, dst), // movzbl
 10249              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
 10250   ins_pipe(pipe_slow); // XXX
 10251 %}
 10253 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
 10254 %{
 10255   match(Set dst (CmpLTMask p q));
 10256   effect(KILL cr);
 10258   ins_cost(400); // XXX
 10259   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
 10260             "setlt   $dst\n\t"
 10261             "movzbl  $dst, $dst\n\t"
 10262             "negl    $dst" %}
 10263   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
 10264              setLT_reg(dst),
 10265              REX_reg_breg(dst, dst), // movzbl
 10266              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
 10267              neg_reg(dst));
 10268   ins_pipe(pipe_slow);
 10269 %}
 10271 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
 10272 %{
 10273   match(Set dst (CmpLTMask dst zero));
 10274   effect(KILL cr);
 10276   ins_cost(100); // XXX
 10277   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
 10278   opcode(0xC1, 0x7);  /* C1 /7 ib */
 10279   ins_encode(reg_opc_imm(dst, 0x1F));
 10280   ins_pipe(ialu_reg);
 10281 %}
 10284 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y,
 10285                          rRegI tmp,
 10286                          rFlagsReg cr)
 10287 %{
 10288   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
 10289   effect(TEMP tmp, KILL cr);
 10291   ins_cost(400); // XXX
 10292   format %{ "subl    $p, $q\t# cadd_cmpLTMask1\n\t"
 10293             "sbbl    $tmp, $tmp\n\t"
 10294             "andl    $tmp, $y\n\t"
 10295             "addl    $p, $tmp" %}
 10296   ins_encode(enc_cmpLTP(p, q, y, tmp));
 10297   ins_pipe(pipe_cmplt);
 10298 %}
 10300 /* If I enable this, I encourage spilling in the inner loop of compress.
 10301 instruct cadd_cmpLTMask_mem( rRegI p, rRegI q, memory y, rRegI tmp, rFlagsReg cr )
 10302 %{
 10303   match(Set p (AddI (AndI (CmpLTMask p q) (LoadI y)) (SubI p q)));
 10304   effect( TEMP tmp, KILL cr );
 10305   ins_cost(400);
 10307   format %{ "SUB    $p,$q\n\t"
 10308             "SBB    RCX,RCX\n\t"
 10309             "AND    RCX,$y\n\t"
 10310             "ADD    $p,RCX" %}
 10311   ins_encode( enc_cmpLTP_mem(p,q,y,tmp) );
 10312 %}
 10313 */
 10315 //---------- FP Instructions------------------------------------------------
 10317 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
 10318 %{
 10319   match(Set cr (CmpF src1 src2));
 10321   ins_cost(145);
 10322   format %{ "ucomiss $src1, $src2\n\t"
 10323             "jnp,s   exit\n\t"
 10324             "pushfq\t# saw NaN, set CF\n\t"
 10325             "andq    [rsp], #0xffffff2b\n\t"
 10326             "popfq\n"
 10327     "exit:   nop\t# avoid branch to branch" %}
 10328   opcode(0x0F, 0x2E);
 10329   ins_encode(REX_reg_reg(src1, src2), OpcP, OpcS, reg_reg(src1, src2),
 10330              cmpfp_fixup);
 10331   ins_pipe(pipe_slow);
 10332 %}
 10334 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
 10335   match(Set cr (CmpF src1 src2));
 10337   ins_cost(145);
 10338   format %{ "ucomiss $src1, $src2" %}
 10339   ins_encode %{
 10340     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 10341   %}
 10342   ins_pipe(pipe_slow);
 10343 %}
 10345 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
 10346 %{
 10347   match(Set cr (CmpF src1 (LoadF src2)));
 10349   ins_cost(145);
 10350   format %{ "ucomiss $src1, $src2\n\t"
 10351             "jnp,s   exit\n\t"
 10352             "pushfq\t# saw NaN, set CF\n\t"
 10353             "andq    [rsp], #0xffffff2b\n\t"
 10354             "popfq\n"
 10355     "exit:   nop\t# avoid branch to branch" %}
 10356   opcode(0x0F, 0x2E);
 10357   ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, reg_mem(src1, src2),
 10358              cmpfp_fixup);
 10359   ins_pipe(pipe_slow);
 10360 %}
 10362 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
 10363   match(Set cr (CmpF src1 (LoadF src2)));
 10365   ins_cost(100);
 10366   format %{ "ucomiss $src1, $src2" %}
 10367   opcode(0x0F, 0x2E);
 10368   ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, reg_mem(src1, src2));
 10369   ins_pipe(pipe_slow);
 10370 %}
 10372 instruct cmpF_cc_imm(rFlagsRegU cr, regF src1, immF src2)
 10373 %{
 10374   match(Set cr (CmpF src1 src2));
 10376   ins_cost(145);
 10377   format %{ "ucomiss $src1, $src2\n\t"
 10378             "jnp,s   exit\n\t"
 10379             "pushfq\t# saw NaN, set CF\n\t"
 10380             "andq    [rsp], #0xffffff2b\n\t"
 10381             "popfq\n"
 10382     "exit:   nop\t# avoid branch to branch" %}
 10383   opcode(0x0F, 0x2E);
 10384   ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, load_immF(src1, src2),
 10385              cmpfp_fixup);
 10386   ins_pipe(pipe_slow);
 10387 %}
 10389 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src1, immF src2) %{
 10390   match(Set cr (CmpF src1 src2));
 10392   ins_cost(100);
 10393   format %{ "ucomiss $src1, $src2" %}
 10394   opcode(0x0F, 0x2E);
 10395   ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, load_immF(src1, src2));
 10396   ins_pipe(pipe_slow);
 10397 %}
 10399 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
 10400 %{
 10401   match(Set cr (CmpD src1 src2));
 10403   ins_cost(145);
 10404   format %{ "ucomisd $src1, $src2\n\t"
 10405             "jnp,s   exit\n\t"
 10406             "pushfq\t# saw NaN, set CF\n\t"
 10407             "andq    [rsp], #0xffffff2b\n\t"
 10408             "popfq\n"
 10409     "exit:   nop\t# avoid branch to branch" %}
 10410   opcode(0x66, 0x0F, 0x2E);
 10411   ins_encode(OpcP, REX_reg_reg(src1, src2), OpcS, OpcT, reg_reg(src1, src2),
 10412              cmpfp_fixup);
 10413   ins_pipe(pipe_slow);
 10414 %}
 10416 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
 10417   match(Set cr (CmpD src1 src2));
 10419   ins_cost(100);
 10420   format %{ "ucomisd $src1, $src2 test" %}
 10421   ins_encode %{
 10422     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 10423   %}
 10424   ins_pipe(pipe_slow);
 10425 %}
 10427 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
 10428 %{
 10429   match(Set cr (CmpD src1 (LoadD src2)));
 10431   ins_cost(145);
 10432   format %{ "ucomisd $src1, $src2\n\t"
 10433             "jnp,s   exit\n\t"
 10434             "pushfq\t# saw NaN, set CF\n\t"
 10435             "andq    [rsp], #0xffffff2b\n\t"
 10436             "popfq\n"
 10437     "exit:   nop\t# avoid branch to branch" %}
 10438   opcode(0x66, 0x0F, 0x2E);
 10439   ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, reg_mem(src1, src2),
 10440              cmpfp_fixup);
 10441   ins_pipe(pipe_slow);
 10442 %}
 10444 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
 10445   match(Set cr (CmpD src1 (LoadD src2)));
 10447   ins_cost(100);
 10448   format %{ "ucomisd $src1, $src2" %}
 10449   opcode(0x66, 0x0F, 0x2E);
 10450   ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, reg_mem(src1, src2));
 10451   ins_pipe(pipe_slow);
 10452 %}
 10454 instruct cmpD_cc_imm(rFlagsRegU cr, regD src1, immD src2)
 10455 %{
 10456   match(Set cr (CmpD src1 src2));
 10458   ins_cost(145);
 10459   format %{ "ucomisd $src1, [$src2]\n\t"
 10460             "jnp,s   exit\n\t"
 10461             "pushfq\t# saw NaN, set CF\n\t"
 10462             "andq    [rsp], #0xffffff2b\n\t"
 10463             "popfq\n"
 10464     "exit:   nop\t# avoid branch to branch" %}
 10465   opcode(0x66, 0x0F, 0x2E);
 10466   ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, load_immD(src1, src2),
 10467              cmpfp_fixup);
 10468   ins_pipe(pipe_slow);
 10469 %}
 10471 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src1, immD src2) %{
 10472   match(Set cr (CmpD src1 src2));
 10474   ins_cost(100);
 10475   format %{ "ucomisd $src1, [$src2]" %}
 10476   opcode(0x66, 0x0F, 0x2E);
 10477   ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, load_immD(src1, src2));
 10478   ins_pipe(pipe_slow);
 10479 %}
 10481 // Compare into -1,0,1
 10482 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
 10483 %{
 10484   match(Set dst (CmpF3 src1 src2));
 10485   effect(KILL cr);
 10487   ins_cost(275);
 10488   format %{ "ucomiss $src1, $src2\n\t"
 10489             "movl    $dst, #-1\n\t"
 10490             "jp,s    done\n\t"
 10491             "jb,s    done\n\t"
 10492             "setne   $dst\n\t"
 10493             "movzbl  $dst, $dst\n"
 10494     "done:" %}
 10496   opcode(0x0F, 0x2E);
 10497   ins_encode(REX_reg_reg(src1, src2), OpcP, OpcS, reg_reg(src1, src2),
 10498              cmpfp3(dst));
 10499   ins_pipe(pipe_slow);
 10500 %}
 10502 // Compare into -1,0,1
 10503 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
 10504 %{
 10505   match(Set dst (CmpF3 src1 (LoadF src2)));
 10506   effect(KILL cr);
 10508   ins_cost(275);
 10509   format %{ "ucomiss $src1, $src2\n\t"
 10510             "movl    $dst, #-1\n\t"
 10511             "jp,s    done\n\t"
 10512             "jb,s    done\n\t"
 10513             "setne   $dst\n\t"
 10514             "movzbl  $dst, $dst\n"
 10515     "done:" %}
 10517   opcode(0x0F, 0x2E);
 10518   ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, reg_mem(src1, src2),
 10519              cmpfp3(dst));
 10520   ins_pipe(pipe_slow);
 10521 %}
 10523 // Compare into -1,0,1
 10524 instruct cmpF_imm(rRegI dst, regF src1, immF src2, rFlagsReg cr)
 10525 %{
 10526   match(Set dst (CmpF3 src1 src2));
 10527   effect(KILL cr);
 10529   ins_cost(275);
 10530   format %{ "ucomiss $src1, [$src2]\n\t"
 10531             "movl    $dst, #-1\n\t"
 10532             "jp,s    done\n\t"
 10533             "jb,s    done\n\t"
 10534             "setne   $dst\n\t"
 10535             "movzbl  $dst, $dst\n"
 10536     "done:" %}
 10538   opcode(0x0F, 0x2E);
 10539   ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, load_immF(src1, src2),
 10540              cmpfp3(dst));
 10541   ins_pipe(pipe_slow);
 10542 %}
 10544 // Compare into -1,0,1
 10545 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
 10546 %{
 10547   match(Set dst (CmpD3 src1 src2));
 10548   effect(KILL cr);
 10550   ins_cost(275);
 10551   format %{ "ucomisd $src1, $src2\n\t"
 10552             "movl    $dst, #-1\n\t"
 10553             "jp,s    done\n\t"
 10554             "jb,s    done\n\t"
 10555             "setne   $dst\n\t"
 10556             "movzbl  $dst, $dst\n"
 10557     "done:" %}
 10559   opcode(0x66, 0x0F, 0x2E);
 10560   ins_encode(OpcP, REX_reg_reg(src1, src2), OpcS, OpcT, reg_reg(src1, src2),
 10561              cmpfp3(dst));
 10562   ins_pipe(pipe_slow);
 10563 %}
 10565 // Compare into -1,0,1
 10566 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
 10567 %{
 10568   match(Set dst (CmpD3 src1 (LoadD src2)));
 10569   effect(KILL cr);
 10571   ins_cost(275);
 10572   format %{ "ucomisd $src1, $src2\n\t"
 10573             "movl    $dst, #-1\n\t"
 10574             "jp,s    done\n\t"
 10575             "jb,s    done\n\t"
 10576             "setne   $dst\n\t"
 10577             "movzbl  $dst, $dst\n"
 10578     "done:" %}
 10580   opcode(0x66, 0x0F, 0x2E);
 10581   ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, reg_mem(src1, src2),
 10582              cmpfp3(dst));
 10583   ins_pipe(pipe_slow);
 10584 %}
 10586 // Compare into -1,0,1
 10587 instruct cmpD_imm(rRegI dst, regD src1, immD src2, rFlagsReg cr)
 10588 %{
 10589   match(Set dst (CmpD3 src1 src2));
 10590   effect(KILL cr);
 10592   ins_cost(275);
 10593   format %{ "ucomisd $src1, [$src2]\n\t"
 10594             "movl    $dst, #-1\n\t"
 10595             "jp,s    done\n\t"
 10596             "jb,s    done\n\t"
 10597             "setne   $dst\n\t"
 10598             "movzbl  $dst, $dst\n"
 10599     "done:" %}
 10601   opcode(0x66, 0x0F, 0x2E);
 10602   ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, load_immD(src1, src2),
 10603              cmpfp3(dst));
 10604   ins_pipe(pipe_slow);
 10605 %}
 10607 instruct addF_reg(regF dst, regF src)
 10608 %{
 10609   match(Set dst (AddF dst src));
 10611   format %{ "addss   $dst, $src" %}
 10612   ins_cost(150); // XXX
 10613   opcode(0xF3, 0x0F, 0x58);
 10614   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 10615   ins_pipe(pipe_slow);
 10616 %}
 10618 instruct addF_mem(regF dst, memory src)
 10619 %{
 10620   match(Set dst (AddF dst (LoadF src)));
 10622   format %{ "addss   $dst, $src" %}
 10623   ins_cost(150); // XXX
 10624   opcode(0xF3, 0x0F, 0x58);
 10625   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 10626   ins_pipe(pipe_slow);
 10627 %}
 10629 instruct addF_imm(regF dst, immF src)
 10630 %{
 10631   match(Set dst (AddF dst src));
 10633   format %{ "addss   $dst, [$src]" %}
 10634   ins_cost(150); // XXX
 10635   opcode(0xF3, 0x0F, 0x58);
 10636   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immF(dst, src));
 10637   ins_pipe(pipe_slow);
 10638 %}
 10640 instruct addD_reg(regD dst, regD src)
 10641 %{
 10642   match(Set dst (AddD dst src));
 10644   format %{ "addsd   $dst, $src" %}
 10645   ins_cost(150); // XXX
 10646   opcode(0xF2, 0x0F, 0x58);
 10647   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 10648   ins_pipe(pipe_slow);
 10649 %}
 10651 instruct addD_mem(regD dst, memory src)
 10652 %{
 10653   match(Set dst (AddD dst (LoadD src)));
 10655   format %{ "addsd   $dst, $src" %}
 10656   ins_cost(150); // XXX
 10657   opcode(0xF2, 0x0F, 0x58);
 10658   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 10659   ins_pipe(pipe_slow);
 10660 %}
 10662 instruct addD_imm(regD dst, immD src)
 10663 %{
 10664   match(Set dst (AddD dst src));
 10666   format %{ "addsd   $dst, [$src]" %}
 10667   ins_cost(150); // XXX
 10668   opcode(0xF2, 0x0F, 0x58);
 10669   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immD(dst, src));
 10670   ins_pipe(pipe_slow);
 10671 %}
 10673 instruct subF_reg(regF dst, regF src)
 10674 %{
 10675   match(Set dst (SubF dst src));
 10677   format %{ "subss   $dst, $src" %}
 10678   ins_cost(150); // XXX
 10679   opcode(0xF3, 0x0F, 0x5C);
 10680   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 10681   ins_pipe(pipe_slow);
 10682 %}
 10684 instruct subF_mem(regF dst, memory src)
 10685 %{
 10686   match(Set dst (SubF dst (LoadF src)));
 10688   format %{ "subss   $dst, $src" %}
 10689   ins_cost(150); // XXX
 10690   opcode(0xF3, 0x0F, 0x5C);
 10691   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 10692   ins_pipe(pipe_slow);
 10693 %}
 10695 instruct subF_imm(regF dst, immF src)
 10696 %{
 10697   match(Set dst (SubF dst src));
 10699   format %{ "subss   $dst, [$src]" %}
 10700   ins_cost(150); // XXX
 10701   opcode(0xF3, 0x0F, 0x5C);
 10702   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immF(dst, src));
 10703   ins_pipe(pipe_slow);
 10704 %}
 10706 instruct subD_reg(regD dst, regD src)
 10707 %{
 10708   match(Set dst (SubD dst src));
 10710   format %{ "subsd   $dst, $src" %}
 10711   ins_cost(150); // XXX
 10712   opcode(0xF2, 0x0F, 0x5C);
 10713   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 10714   ins_pipe(pipe_slow);
 10715 %}
 10717 instruct subD_mem(regD dst, memory src)
 10718 %{
 10719   match(Set dst (SubD dst (LoadD src)));
 10721   format %{ "subsd   $dst, $src" %}
 10722   ins_cost(150); // XXX
 10723   opcode(0xF2, 0x0F, 0x5C);
 10724   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 10725   ins_pipe(pipe_slow);
 10726 %}
 10728 instruct subD_imm(regD dst, immD src)
 10729 %{
 10730   match(Set dst (SubD dst src));
 10732   format %{ "subsd   $dst, [$src]" %}
 10733   ins_cost(150); // XXX
 10734   opcode(0xF2, 0x0F, 0x5C);
 10735   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immD(dst, src));
 10736   ins_pipe(pipe_slow);
 10737 %}
 10739 instruct mulF_reg(regF dst, regF src)
 10740 %{
 10741   match(Set dst (MulF dst src));
 10743   format %{ "mulss   $dst, $src" %}
 10744   ins_cost(150); // XXX
 10745   opcode(0xF3, 0x0F, 0x59);
 10746   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 10747   ins_pipe(pipe_slow);
 10748 %}
 10750 instruct mulF_mem(regF dst, memory src)
 10751 %{
 10752   match(Set dst (MulF dst (LoadF src)));
 10754   format %{ "mulss   $dst, $src" %}
 10755   ins_cost(150); // XXX
 10756   opcode(0xF3, 0x0F, 0x59);
 10757   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 10758   ins_pipe(pipe_slow);
 10759 %}
 10761 instruct mulF_imm(regF dst, immF src)
 10762 %{
 10763   match(Set dst (MulF dst src));
 10765   format %{ "mulss   $dst, [$src]" %}
 10766   ins_cost(150); // XXX
 10767   opcode(0xF3, 0x0F, 0x59);
 10768   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immF(dst, src));
 10769   ins_pipe(pipe_slow);
 10770 %}
 10772 instruct mulD_reg(regD dst, regD src)
 10773 %{
 10774   match(Set dst (MulD dst src));
 10776   format %{ "mulsd   $dst, $src" %}
 10777   ins_cost(150); // XXX
 10778   opcode(0xF2, 0x0F, 0x59);
 10779   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 10780   ins_pipe(pipe_slow);
 10781 %}
 10783 instruct mulD_mem(regD dst, memory src)
 10784 %{
 10785   match(Set dst (MulD dst (LoadD src)));
 10787   format %{ "mulsd   $dst, $src" %}
 10788   ins_cost(150); // XXX
 10789   opcode(0xF2, 0x0F, 0x59);
 10790   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 10791   ins_pipe(pipe_slow);
 10792 %}
 10794 instruct mulD_imm(regD dst, immD src)
 10795 %{
 10796   match(Set dst (MulD dst src));
 10798   format %{ "mulsd   $dst, [$src]" %}
 10799   ins_cost(150); // XXX
 10800   opcode(0xF2, 0x0F, 0x59);
 10801   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immD(dst, src));
 10802   ins_pipe(pipe_slow);
 10803 %}
 10805 instruct divF_reg(regF dst, regF src)
 10806 %{
 10807   match(Set dst (DivF dst src));
 10809   format %{ "divss   $dst, $src" %}
 10810   ins_cost(150); // XXX
 10811   opcode(0xF3, 0x0F, 0x5E);
 10812   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 10813   ins_pipe(pipe_slow);
 10814 %}
 10816 instruct divF_mem(regF dst, memory src)
 10817 %{
 10818   match(Set dst (DivF dst (LoadF src)));
 10820   format %{ "divss   $dst, $src" %}
 10821   ins_cost(150); // XXX
 10822   opcode(0xF3, 0x0F, 0x5E);
 10823   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 10824   ins_pipe(pipe_slow);
 10825 %}
 10827 instruct divF_imm(regF dst, immF src)
 10828 %{
 10829   match(Set dst (DivF dst src));
 10831   format %{ "divss   $dst, [$src]" %}
 10832   ins_cost(150); // XXX
 10833   opcode(0xF3, 0x0F, 0x5E);
 10834   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immF(dst, src));
 10835   ins_pipe(pipe_slow);
 10836 %}
 10838 instruct divD_reg(regD dst, regD src)
 10839 %{
 10840   match(Set dst (DivD dst src));
 10842   format %{ "divsd   $dst, $src" %}
 10843   ins_cost(150); // XXX
 10844   opcode(0xF2, 0x0F, 0x5E);
 10845   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 10846   ins_pipe(pipe_slow);
 10847 %}
 10849 instruct divD_mem(regD dst, memory src)
 10850 %{
 10851   match(Set dst (DivD dst (LoadD src)));
 10853   format %{ "divsd   $dst, $src" %}
 10854   ins_cost(150); // XXX
 10855   opcode(0xF2, 0x0F, 0x5E);
 10856   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 10857   ins_pipe(pipe_slow);
 10858 %}
 10860 instruct divD_imm(regD dst, immD src)
 10861 %{
 10862   match(Set dst (DivD dst src));
 10864   format %{ "divsd   $dst, [$src]" %}
 10865   ins_cost(150); // XXX
 10866   opcode(0xF2, 0x0F, 0x5E);
 10867   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immD(dst, src));
 10868   ins_pipe(pipe_slow);
 10869 %}
 10871 instruct sqrtF_reg(regF dst, regF src)
 10872 %{
 10873   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
 10875   format %{ "sqrtss  $dst, $src" %}
 10876   ins_cost(150); // XXX
 10877   opcode(0xF3, 0x0F, 0x51);
 10878   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 10879   ins_pipe(pipe_slow);
 10880 %}
 10882 instruct sqrtF_mem(regF dst, memory src)
 10883 %{
 10884   match(Set dst (ConvD2F (SqrtD (ConvF2D (LoadF src)))));
 10886   format %{ "sqrtss  $dst, $src" %}
 10887   ins_cost(150); // XXX
 10888   opcode(0xF3, 0x0F, 0x51);
 10889   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 10890   ins_pipe(pipe_slow);
 10891 %}
 10893 instruct sqrtF_imm(regF dst, immF src)
 10894 %{
 10895   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
 10897   format %{ "sqrtss  $dst, [$src]" %}
 10898   ins_cost(150); // XXX
 10899   opcode(0xF3, 0x0F, 0x51);
 10900   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immF(dst, src));
 10901   ins_pipe(pipe_slow);
 10902 %}
 10904 instruct sqrtD_reg(regD dst, regD src)
 10905 %{
 10906   match(Set dst (SqrtD src));
 10908   format %{ "sqrtsd  $dst, $src" %}
 10909   ins_cost(150); // XXX
 10910   opcode(0xF2, 0x0F, 0x51);
 10911   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 10912   ins_pipe(pipe_slow);
 10913 %}
 10915 instruct sqrtD_mem(regD dst, memory src)
 10916 %{
 10917   match(Set dst (SqrtD (LoadD src)));
 10919   format %{ "sqrtsd  $dst, $src" %}
 10920   ins_cost(150); // XXX
 10921   opcode(0xF2, 0x0F, 0x51);
 10922   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 10923   ins_pipe(pipe_slow);
 10924 %}
 10926 instruct sqrtD_imm(regD dst, immD src)
 10927 %{
 10928   match(Set dst (SqrtD src));
 10930   format %{ "sqrtsd  $dst, [$src]" %}
 10931   ins_cost(150); // XXX
 10932   opcode(0xF2, 0x0F, 0x51);
 10933   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, load_immD(dst, src));
 10934   ins_pipe(pipe_slow);
 10935 %}
 10937 instruct absF_reg(regF dst)
 10938 %{
 10939   match(Set dst (AbsF dst));
 10941   format %{ "andps   $dst, [0x7fffffff]\t# abs float by sign masking" %}
 10942   ins_encode(absF_encoding(dst));
 10943   ins_pipe(pipe_slow);
 10944 %}
 10946 instruct absD_reg(regD dst)
 10947 %{
 10948   match(Set dst (AbsD dst));
 10950   format %{ "andpd   $dst, [0x7fffffffffffffff]\t"
 10951             "# abs double by sign masking" %}
 10952   ins_encode(absD_encoding(dst));
 10953   ins_pipe(pipe_slow);
 10954 %}
 10956 instruct negF_reg(regF dst)
 10957 %{
 10958   match(Set dst (NegF dst));
 10960   format %{ "xorps   $dst, [0x80000000]\t# neg float by sign flipping" %}
 10961   ins_encode(negF_encoding(dst));
 10962   ins_pipe(pipe_slow);
 10963 %}
 10965 instruct negD_reg(regD dst)
 10966 %{
 10967   match(Set dst (NegD dst));
 10969   format %{ "xorpd   $dst, [0x8000000000000000]\t"
 10970             "# neg double by sign flipping" %}
 10971   ins_encode(negD_encoding(dst));
 10972   ins_pipe(pipe_slow);
 10973 %}
 10975 // -----------Trig and Trancendental Instructions------------------------------
 10976 instruct cosD_reg(regD dst) %{
 10977   match(Set dst (CosD dst));
 10979   format %{ "dcos   $dst\n\t" %}
 10980   opcode(0xD9, 0xFF);
 10981   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
 10982   ins_pipe( pipe_slow );
 10983 %}
 10985 instruct sinD_reg(regD dst) %{
 10986   match(Set dst (SinD dst));
 10988   format %{ "dsin   $dst\n\t" %}
 10989   opcode(0xD9, 0xFE);
 10990   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
 10991   ins_pipe( pipe_slow );
 10992 %}
 10994 instruct tanD_reg(regD dst) %{
 10995   match(Set dst (TanD dst));
 10997   format %{ "dtan   $dst\n\t" %}
 10998   ins_encode( Push_SrcXD(dst),
 10999               Opcode(0xD9), Opcode(0xF2),   //fptan
 11000               Opcode(0xDD), Opcode(0xD8),   //fstp st
 11001               Push_ResultXD(dst) );
 11002   ins_pipe( pipe_slow );
 11003 %}
 11005 instruct log10D_reg(regD dst) %{
 11006   // The source and result Double operands in XMM registers
 11007   match(Set dst (Log10D dst));
 11008   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
 11009   // fyl2x        ; compute log_10(2) * log_2(x)
 11010   format %{ "fldlg2\t\t\t#Log10\n\t"
 11011             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
 11012          %}
 11013    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
 11014               Push_SrcXD(dst),
 11015               Opcode(0xD9), Opcode(0xF1),   // fyl2x
 11016               Push_ResultXD(dst));
 11018   ins_pipe( pipe_slow );
 11019 %}
 11021 instruct logD_reg(regD dst) %{
 11022   // The source and result Double operands in XMM registers
 11023   match(Set dst (LogD dst));
 11024   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
 11025   // fyl2x        ; compute log_e(2) * log_2(x)
 11026   format %{ "fldln2\t\t\t#Log_e\n\t"
 11027             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
 11028          %}
 11029   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
 11030               Push_SrcXD(dst),
 11031               Opcode(0xD9), Opcode(0xF1),   // fyl2x
 11032               Push_ResultXD(dst));
 11033   ins_pipe( pipe_slow );
 11034 %}
 11038 //----------Arithmetic Conversion Instructions---------------------------------
 11040 instruct roundFloat_nop(regF dst)
 11041 %{
 11042   match(Set dst (RoundFloat dst));
 11044   ins_cost(0);
 11045   ins_encode();
 11046   ins_pipe(empty);
 11047 %}
 11049 instruct roundDouble_nop(regD dst)
 11050 %{
 11051   match(Set dst (RoundDouble dst));
 11053   ins_cost(0);
 11054   ins_encode();
 11055   ins_pipe(empty);
 11056 %}
 11058 instruct convF2D_reg_reg(regD dst, regF src)
 11059 %{
 11060   match(Set dst (ConvF2D src));
 11062   format %{ "cvtss2sd $dst, $src" %}
 11063   opcode(0xF3, 0x0F, 0x5A);
 11064   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 11065   ins_pipe(pipe_slow); // XXX
 11066 %}
 11068 instruct convF2D_reg_mem(regD dst, memory src)
 11069 %{
 11070   match(Set dst (ConvF2D (LoadF src)));
 11072   format %{ "cvtss2sd $dst, $src" %}
 11073   opcode(0xF3, 0x0F, 0x5A);
 11074   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 11075   ins_pipe(pipe_slow); // XXX
 11076 %}
 11078 instruct convD2F_reg_reg(regF dst, regD src)
 11079 %{
 11080   match(Set dst (ConvD2F src));
 11082   format %{ "cvtsd2ss $dst, $src" %}
 11083   opcode(0xF2, 0x0F, 0x5A);
 11084   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 11085   ins_pipe(pipe_slow); // XXX
 11086 %}
 11088 instruct convD2F_reg_mem(regF dst, memory src)
 11089 %{
 11090   match(Set dst (ConvD2F (LoadD src)));
 11092   format %{ "cvtsd2ss $dst, $src" %}
 11093   opcode(0xF2, 0x0F, 0x5A);
 11094   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 11095   ins_pipe(pipe_slow); // XXX
 11096 %}
 11098 // XXX do mem variants
 11099 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
 11100 %{
 11101   match(Set dst (ConvF2I src));
 11102   effect(KILL cr);
 11104   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
 11105             "cmpl    $dst, #0x80000000\n\t"
 11106             "jne,s   done\n\t"
 11107             "subq    rsp, #8\n\t"
 11108             "movss   [rsp], $src\n\t"
 11109             "call    f2i_fixup\n\t"
 11110             "popq    $dst\n"
 11111     "done:   "%}
 11112   opcode(0xF3, 0x0F, 0x2C);
 11113   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src),
 11114              f2i_fixup(dst, src));
 11115   ins_pipe(pipe_slow);
 11116 %}
 11118 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
 11119 %{
 11120   match(Set dst (ConvF2L src));
 11121   effect(KILL cr);
 11123   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
 11124             "cmpq    $dst, [0x8000000000000000]\n\t"
 11125             "jne,s   done\n\t"
 11126             "subq    rsp, #8\n\t"
 11127             "movss   [rsp], $src\n\t"
 11128             "call    f2l_fixup\n\t"
 11129             "popq    $dst\n"
 11130     "done:   "%}
 11131   opcode(0xF3, 0x0F, 0x2C);
 11132   ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src),
 11133              f2l_fixup(dst, src));
 11134   ins_pipe(pipe_slow);
 11135 %}
 11137 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
 11138 %{
 11139   match(Set dst (ConvD2I src));
 11140   effect(KILL cr);
 11142   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
 11143             "cmpl    $dst, #0x80000000\n\t"
 11144             "jne,s   done\n\t"
 11145             "subq    rsp, #8\n\t"
 11146             "movsd   [rsp], $src\n\t"
 11147             "call    d2i_fixup\n\t"
 11148             "popq    $dst\n"
 11149     "done:   "%}
 11150   opcode(0xF2, 0x0F, 0x2C);
 11151   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src),
 11152              d2i_fixup(dst, src));
 11153   ins_pipe(pipe_slow);
 11154 %}
 11156 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
 11157 %{
 11158   match(Set dst (ConvD2L src));
 11159   effect(KILL cr);
 11161   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
 11162             "cmpq    $dst, [0x8000000000000000]\n\t"
 11163             "jne,s   done\n\t"
 11164             "subq    rsp, #8\n\t"
 11165             "movsd   [rsp], $src\n\t"
 11166             "call    d2l_fixup\n\t"
 11167             "popq    $dst\n"
 11168     "done:   "%}
 11169   opcode(0xF2, 0x0F, 0x2C);
 11170   ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src),
 11171              d2l_fixup(dst, src));
 11172   ins_pipe(pipe_slow);
 11173 %}
 11175 instruct convI2F_reg_reg(regF dst, rRegI src)
 11176 %{
 11177   predicate(!UseXmmI2F);
 11178   match(Set dst (ConvI2F src));
 11180   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
 11181   opcode(0xF3, 0x0F, 0x2A);
 11182   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 11183   ins_pipe(pipe_slow); // XXX
 11184 %}
 11186 instruct convI2F_reg_mem(regF dst, memory src)
 11187 %{
 11188   match(Set dst (ConvI2F (LoadI src)));
 11190   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
 11191   opcode(0xF3, 0x0F, 0x2A);
 11192   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 11193   ins_pipe(pipe_slow); // XXX
 11194 %}
 11196 instruct convI2D_reg_reg(regD dst, rRegI src)
 11197 %{
 11198   predicate(!UseXmmI2D);
 11199   match(Set dst (ConvI2D src));
 11201   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
 11202   opcode(0xF2, 0x0F, 0x2A);
 11203   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
 11204   ins_pipe(pipe_slow); // XXX
 11205 %}
 11207 instruct convI2D_reg_mem(regD dst, memory src)
 11208 %{
 11209   match(Set dst (ConvI2D (LoadI src)));
 11211   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
 11212   opcode(0xF2, 0x0F, 0x2A);
 11213   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 11214   ins_pipe(pipe_slow); // XXX
 11215 %}
 11217 instruct convXI2F_reg(regF dst, rRegI src)
 11218 %{
 11219   predicate(UseXmmI2F);
 11220   match(Set dst (ConvI2F src));
 11222   format %{ "movdl $dst, $src\n\t"
 11223             "cvtdq2psl $dst, $dst\t# i2f" %}
 11224   ins_encode %{
 11225     __ movdl($dst$$XMMRegister, $src$$Register);
 11226     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
 11227   %}
 11228   ins_pipe(pipe_slow); // XXX
 11229 %}
 11231 instruct convXI2D_reg(regD dst, rRegI src)
 11232 %{
 11233   predicate(UseXmmI2D);
 11234   match(Set dst (ConvI2D src));
 11236   format %{ "movdl $dst, $src\n\t"
 11237             "cvtdq2pdl $dst, $dst\t# i2d" %}
 11238   ins_encode %{
 11239     __ movdl($dst$$XMMRegister, $src$$Register);
 11240     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
 11241   %}
 11242   ins_pipe(pipe_slow); // XXX
 11243 %}
 11245 instruct convL2F_reg_reg(regF dst, rRegL src)
 11246 %{
 11247   match(Set dst (ConvL2F src));
 11249   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
 11250   opcode(0xF3, 0x0F, 0x2A);
 11251   ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src));
 11252   ins_pipe(pipe_slow); // XXX
 11253 %}
 11255 instruct convL2F_reg_mem(regF dst, memory src)
 11256 %{
 11257   match(Set dst (ConvL2F (LoadL src)));
 11259   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
 11260   opcode(0xF3, 0x0F, 0x2A);
 11261   ins_encode(OpcP, REX_reg_mem_wide(dst, src), OpcS, OpcT, reg_mem(dst, src));
 11262   ins_pipe(pipe_slow); // XXX
 11263 %}
 11265 instruct convL2D_reg_reg(regD dst, rRegL src)
 11266 %{
 11267   match(Set dst (ConvL2D src));
 11269   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
 11270   opcode(0xF2, 0x0F, 0x2A);
 11271   ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src));
 11272   ins_pipe(pipe_slow); // XXX
 11273 %}
 11275 instruct convL2D_reg_mem(regD dst, memory src)
 11276 %{
 11277   match(Set dst (ConvL2D (LoadL src)));
 11279   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
 11280   opcode(0xF2, 0x0F, 0x2A);
 11281   ins_encode(OpcP, REX_reg_mem_wide(dst, src), OpcS, OpcT, reg_mem(dst, src));
 11282   ins_pipe(pipe_slow); // XXX
 11283 %}
 11285 instruct convI2L_reg_reg(rRegL dst, rRegI src)
 11286 %{
 11287   match(Set dst (ConvI2L src));
 11289   ins_cost(125);
 11290   format %{ "movslq  $dst, $src\t# i2l" %}
 11291   ins_encode %{
 11292     __ movslq($dst$$Register, $src$$Register);
 11293   %}
 11294   ins_pipe(ialu_reg_reg);
 11295 %}
 11297 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
 11298 // %{
 11299 //   match(Set dst (ConvI2L src));
 11300 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
 11301 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
 11302 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
 11303 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
 11304 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
 11305 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
 11307 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
 11308 //   ins_encode(enc_copy(dst, src));
 11309 // //   opcode(0x63); // needs REX.W
 11310 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
 11311 //   ins_pipe(ialu_reg_reg);
 11312 // %}
 11314 // Zero-extend convert int to long
 11315 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
 11316 %{
 11317   match(Set dst (AndL (ConvI2L src) mask));
 11319   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
 11320   ins_encode(enc_copy(dst, src));
 11321   ins_pipe(ialu_reg_reg);
 11322 %}
 11324 // Zero-extend convert int to long
 11325 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
 11326 %{
 11327   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
 11329   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
 11330   opcode(0x8B);
 11331   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 11332   ins_pipe(ialu_reg_mem);
 11333 %}
 11335 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
 11336 %{
 11337   match(Set dst (AndL src mask));
 11339   format %{ "movl    $dst, $src\t# zero-extend long" %}
 11340   ins_encode(enc_copy_always(dst, src));
 11341   ins_pipe(ialu_reg_reg);
 11342 %}
 11344 instruct convL2I_reg_reg(rRegI dst, rRegL src)
 11345 %{
 11346   match(Set dst (ConvL2I src));
 11348   format %{ "movl    $dst, $src\t# l2i" %}
 11349   ins_encode(enc_copy_always(dst, src));
 11350   ins_pipe(ialu_reg_reg);
 11351 %}
 11354 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
 11355   match(Set dst (MoveF2I src));
 11356   effect(DEF dst, USE src);
 11358   ins_cost(125);
 11359   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
 11360   opcode(0x8B);
 11361   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 11362   ins_pipe(ialu_reg_mem);
 11363 %}
 11365 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
 11366   match(Set dst (MoveI2F src));
 11367   effect(DEF dst, USE src);
 11369   ins_cost(125);
 11370   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
 11371   opcode(0xF3, 0x0F, 0x10);
 11372   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 11373   ins_pipe(pipe_slow);
 11374 %}
 11376 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
 11377   match(Set dst (MoveD2L src));
 11378   effect(DEF dst, USE src);
 11380   ins_cost(125);
 11381   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
 11382   opcode(0x8B);
 11383   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 11384   ins_pipe(ialu_reg_mem);
 11385 %}
 11387 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
 11388   predicate(!UseXmmLoadAndClearUpper);
 11389   match(Set dst (MoveL2D src));
 11390   effect(DEF dst, USE src);
 11392   ins_cost(125);
 11393   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
 11394   opcode(0x66, 0x0F, 0x12);
 11395   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 11396   ins_pipe(pipe_slow);
 11397 %}
 11399 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
 11400   predicate(UseXmmLoadAndClearUpper);
 11401   match(Set dst (MoveL2D src));
 11402   effect(DEF dst, USE src);
 11404   ins_cost(125);
 11405   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
 11406   opcode(0xF2, 0x0F, 0x10);
 11407   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
 11408   ins_pipe(pipe_slow);
 11409 %}
 11412 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
 11413   match(Set dst (MoveF2I src));
 11414   effect(DEF dst, USE src);
 11416   ins_cost(95); // XXX
 11417   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
 11418   opcode(0xF3, 0x0F, 0x11);
 11419   ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
 11420   ins_pipe(pipe_slow);
 11421 %}
 11423 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
 11424   match(Set dst (MoveI2F src));
 11425   effect(DEF dst, USE src);
 11427   ins_cost(100);
 11428   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
 11429   opcode(0x89);
 11430   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 11431   ins_pipe( ialu_mem_reg );
 11432 %}
 11434 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
 11435   match(Set dst (MoveD2L src));
 11436   effect(DEF dst, USE src);
 11438   ins_cost(95); // XXX
 11439   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
 11440   opcode(0xF2, 0x0F, 0x11);
 11441   ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
 11442   ins_pipe(pipe_slow);
 11443 %}
 11445 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
 11446   match(Set dst (MoveL2D src));
 11447   effect(DEF dst, USE src);
 11449   ins_cost(100);
 11450   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
 11451   opcode(0x89);
 11452   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 11453   ins_pipe(ialu_mem_reg);
 11454 %}
 11456 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
 11457   match(Set dst (MoveF2I src));
 11458   effect(DEF dst, USE src);
 11459   ins_cost(85);
 11460   format %{ "movd    $dst,$src\t# MoveF2I" %}
 11461   ins_encode %{ __ movdl($dst$$Register, $src$$XMMRegister); %}
 11462   ins_pipe( pipe_slow );
 11463 %}
 11465 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
 11466   match(Set dst (MoveD2L src));
 11467   effect(DEF dst, USE src);
 11468   ins_cost(85);
 11469   format %{ "movd    $dst,$src\t# MoveD2L" %}
 11470   ins_encode %{ __ movdq($dst$$Register, $src$$XMMRegister); %}
 11471   ins_pipe( pipe_slow );
 11472 %}
 11474 // The next instructions have long latency and use Int unit. Set high cost.
 11475 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
 11476   match(Set dst (MoveI2F src));
 11477   effect(DEF dst, USE src);
 11478   ins_cost(300);
 11479   format %{ "movd    $dst,$src\t# MoveI2F" %}
 11480   ins_encode %{ __ movdl($dst$$XMMRegister, $src$$Register); %}
 11481   ins_pipe( pipe_slow );
 11482 %}
 11484 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
 11485   match(Set dst (MoveL2D src));
 11486   effect(DEF dst, USE src);
 11487   ins_cost(300);
 11488   format %{ "movd    $dst,$src\t# MoveL2D" %}
 11489   ins_encode %{ __ movdq($dst$$XMMRegister, $src$$Register); %}
 11490   ins_pipe( pipe_slow );
 11491 %}
 11493 // Replicate scalar to packed byte (1 byte) values in xmm
 11494 instruct Repl8B_reg(regD dst, regD src) %{
 11495   match(Set dst (Replicate8B src));
 11496   format %{ "MOVDQA  $dst,$src\n\t"
 11497             "PUNPCKLBW $dst,$dst\n\t"
 11498             "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
 11499   ins_encode( pshufd_8x8(dst, src));
 11500   ins_pipe( pipe_slow );
 11501 %}
 11503 // Replicate scalar to packed byte (1 byte) values in xmm
 11504 instruct Repl8B_rRegI(regD dst, rRegI src) %{
 11505   match(Set dst (Replicate8B src));
 11506   format %{ "MOVD    $dst,$src\n\t"
 11507             "PUNPCKLBW $dst,$dst\n\t"
 11508             "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
 11509   ins_encode( mov_i2x(dst, src), pshufd_8x8(dst, dst));
 11510   ins_pipe( pipe_slow );
 11511 %}
 11513 // Replicate scalar zero to packed byte (1 byte) values in xmm
 11514 instruct Repl8B_immI0(regD dst, immI0 zero) %{
 11515   match(Set dst (Replicate8B zero));
 11516   format %{ "PXOR  $dst,$dst\t! replicate8B" %}
 11517   ins_encode( pxor(dst, dst));
 11518   ins_pipe( fpu_reg_reg );
 11519 %}
 11521 // Replicate scalar to packed shore (2 byte) values in xmm
 11522 instruct Repl4S_reg(regD dst, regD src) %{
 11523   match(Set dst (Replicate4S src));
 11524   format %{ "PSHUFLW $dst,$src,0x00\t! replicate4S" %}
 11525   ins_encode( pshufd_4x16(dst, src));
 11526   ins_pipe( fpu_reg_reg );
 11527 %}
 11529 // Replicate scalar to packed shore (2 byte) values in xmm
 11530 instruct Repl4S_rRegI(regD dst, rRegI src) %{
 11531   match(Set dst (Replicate4S src));
 11532   format %{ "MOVD    $dst,$src\n\t"
 11533             "PSHUFLW $dst,$dst,0x00\t! replicate4S" %}
 11534   ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
 11535   ins_pipe( fpu_reg_reg );
 11536 %}
 11538 // Replicate scalar zero to packed short (2 byte) values in xmm
 11539 instruct Repl4S_immI0(regD dst, immI0 zero) %{
 11540   match(Set dst (Replicate4S zero));
 11541   format %{ "PXOR  $dst,$dst\t! replicate4S" %}
 11542   ins_encode( pxor(dst, dst));
 11543   ins_pipe( fpu_reg_reg );
 11544 %}
 11546 // Replicate scalar to packed char (2 byte) values in xmm
 11547 instruct Repl4C_reg(regD dst, regD src) %{
 11548   match(Set dst (Replicate4C src));
 11549   format %{ "PSHUFLW $dst,$src,0x00\t! replicate4C" %}
 11550   ins_encode( pshufd_4x16(dst, src));
 11551   ins_pipe( fpu_reg_reg );
 11552 %}
 11554 // Replicate scalar to packed char (2 byte) values in xmm
 11555 instruct Repl4C_rRegI(regD dst, rRegI src) %{
 11556   match(Set dst (Replicate4C src));
 11557   format %{ "MOVD    $dst,$src\n\t"
 11558             "PSHUFLW $dst,$dst,0x00\t! replicate4C" %}
 11559   ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
 11560   ins_pipe( fpu_reg_reg );
 11561 %}
 11563 // Replicate scalar zero to packed char (2 byte) values in xmm
 11564 instruct Repl4C_immI0(regD dst, immI0 zero) %{
 11565   match(Set dst (Replicate4C zero));
 11566   format %{ "PXOR  $dst,$dst\t! replicate4C" %}
 11567   ins_encode( pxor(dst, dst));
 11568   ins_pipe( fpu_reg_reg );
 11569 %}
 11571 // Replicate scalar to packed integer (4 byte) values in xmm
 11572 instruct Repl2I_reg(regD dst, regD src) %{
 11573   match(Set dst (Replicate2I src));
 11574   format %{ "PSHUFD $dst,$src,0x00\t! replicate2I" %}
 11575   ins_encode( pshufd(dst, src, 0x00));
 11576   ins_pipe( fpu_reg_reg );
 11577 %}
 11579 // Replicate scalar to packed integer (4 byte) values in xmm
 11580 instruct Repl2I_rRegI(regD dst, rRegI src) %{
 11581   match(Set dst (Replicate2I src));
 11582   format %{ "MOVD   $dst,$src\n\t"
 11583             "PSHUFD $dst,$dst,0x00\t! replicate2I" %}
 11584   ins_encode( mov_i2x(dst, src), pshufd(dst, dst, 0x00));
 11585   ins_pipe( fpu_reg_reg );
 11586 %}
 11588 // Replicate scalar zero to packed integer (2 byte) values in xmm
 11589 instruct Repl2I_immI0(regD dst, immI0 zero) %{
 11590   match(Set dst (Replicate2I zero));
 11591   format %{ "PXOR  $dst,$dst\t! replicate2I" %}
 11592   ins_encode( pxor(dst, dst));
 11593   ins_pipe( fpu_reg_reg );
 11594 %}
 11596 // Replicate scalar to packed single precision floating point values in xmm
 11597 instruct Repl2F_reg(regD dst, regD src) %{
 11598   match(Set dst (Replicate2F src));
 11599   format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
 11600   ins_encode( pshufd(dst, src, 0xe0));
 11601   ins_pipe( fpu_reg_reg );
 11602 %}
 11604 // Replicate scalar to packed single precision floating point values in xmm
 11605 instruct Repl2F_regF(regD dst, regF src) %{
 11606   match(Set dst (Replicate2F src));
 11607   format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
 11608   ins_encode( pshufd(dst, src, 0xe0));
 11609   ins_pipe( fpu_reg_reg );
 11610 %}
 11612 // Replicate scalar to packed single precision floating point values in xmm
 11613 instruct Repl2F_immF0(regD dst, immF0 zero) %{
 11614   match(Set dst (Replicate2F zero));
 11615   format %{ "PXOR  $dst,$dst\t! replicate2F" %}
 11616   ins_encode( pxor(dst, dst));
 11617   ins_pipe( fpu_reg_reg );
 11618 %}
 11621 // =======================================================================
 11622 // fast clearing of an array
 11623 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
 11624                   rFlagsReg cr)
 11625 %{
 11626   match(Set dummy (ClearArray cnt base));
 11627   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
 11629   format %{ "xorl    rax, rax\t# ClearArray:\n\t"
 11630             "rep stosq\t# Store rax to *rdi++ while rcx--" %}
 11631   ins_encode(opc_reg_reg(0x33, RAX, RAX), // xorl %eax, %eax
 11632              Opcode(0xF3), Opcode(0x48), Opcode(0xAB)); // rep REX_W stos
 11633   ins_pipe(pipe_slow);
 11634 %}
 11636 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rbx_RegI cnt2,
 11637                         rax_RegI result, regD tmp1, regD tmp2, rFlagsReg cr)
 11638 %{
 11639   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
 11640   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
 11642   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1, $tmp2" %}
 11643   ins_encode %{
 11644     __ string_compare($str1$$Register, $str2$$Register,
 11645                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
 11646                       $tmp1$$XMMRegister, $tmp2$$XMMRegister);
 11647   %}
 11648   ins_pipe( pipe_slow );
 11649 %}
 11651 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
 11652                         rbx_RegI result, regD tmp1, rcx_RegI tmp2, rFlagsReg cr)
 11653 %{
 11654   predicate(UseSSE42Intrinsics);
 11655   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
 11656   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp2, KILL cr);
 11658   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1, $tmp2" %}
 11659   ins_encode %{
 11660     __ string_indexof($str1$$Register, $str2$$Register,
 11661                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
 11662                       $tmp1$$XMMRegister, $tmp2$$Register);
 11663   %}
 11664   ins_pipe( pipe_slow );
 11665 %}
 11667 // fast string equals
 11668 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
 11669                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
 11670 %{
 11671   match(Set result (StrEquals (Binary str1 str2) cnt));
 11672   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
 11674   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
 11675   ins_encode %{
 11676     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
 11677                           $cnt$$Register, $result$$Register, $tmp3$$Register,
 11678                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
 11679   %}
 11680   ins_pipe( pipe_slow );
 11681 %}
 11683 // fast array equals
 11684 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
 11685                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
 11686 %{
 11687   match(Set result (AryEq ary1 ary2));
 11688   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
 11689   //ins_cost(300);
 11691   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
 11692   ins_encode %{
 11693     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
 11694                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
 11695                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
 11696   %}
 11697   ins_pipe( pipe_slow );
 11698 %}
 11700 //----------Control Flow Instructions------------------------------------------
 11701 // Signed compare Instructions
 11703 // XXX more variants!!
 11704 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
 11705 %{
 11706   match(Set cr (CmpI op1 op2));
 11707   effect(DEF cr, USE op1, USE op2);
 11709   format %{ "cmpl    $op1, $op2" %}
 11710   opcode(0x3B);  /* Opcode 3B /r */
 11711   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
 11712   ins_pipe(ialu_cr_reg_reg);
 11713 %}
 11715 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
 11716 %{
 11717   match(Set cr (CmpI op1 op2));
 11719   format %{ "cmpl    $op1, $op2" %}
 11720   opcode(0x81, 0x07); /* Opcode 81 /7 */
 11721   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
 11722   ins_pipe(ialu_cr_reg_imm);
 11723 %}
 11725 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
 11726 %{
 11727   match(Set cr (CmpI op1 (LoadI op2)));
 11729   ins_cost(500); // XXX
 11730   format %{ "cmpl    $op1, $op2" %}
 11731   opcode(0x3B); /* Opcode 3B /r */
 11732   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
 11733   ins_pipe(ialu_cr_reg_mem);
 11734 %}
 11736 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
 11737 %{
 11738   match(Set cr (CmpI src zero));
 11740   format %{ "testl   $src, $src" %}
 11741   opcode(0x85);
 11742   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
 11743   ins_pipe(ialu_cr_reg_imm);
 11744 %}
 11746 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
 11747 %{
 11748   match(Set cr (CmpI (AndI src con) zero));
 11750   format %{ "testl   $src, $con" %}
 11751   opcode(0xF7, 0x00);
 11752   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
 11753   ins_pipe(ialu_cr_reg_imm);
 11754 %}
 11756 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
 11757 %{
 11758   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
 11760   format %{ "testl   $src, $mem" %}
 11761   opcode(0x85);
 11762   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
 11763   ins_pipe(ialu_cr_reg_mem);
 11764 %}
 11766 // Unsigned compare Instructions; really, same as signed except they
 11767 // produce an rFlagsRegU instead of rFlagsReg.
 11768 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
 11769 %{
 11770   match(Set cr (CmpU op1 op2));
 11772   format %{ "cmpl    $op1, $op2\t# unsigned" %}
 11773   opcode(0x3B); /* Opcode 3B /r */
 11774   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
 11775   ins_pipe(ialu_cr_reg_reg);
 11776 %}
 11778 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
 11779 %{
 11780   match(Set cr (CmpU op1 op2));
 11782   format %{ "cmpl    $op1, $op2\t# unsigned" %}
 11783   opcode(0x81,0x07); /* Opcode 81 /7 */
 11784   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
 11785   ins_pipe(ialu_cr_reg_imm);
 11786 %}
 11788 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
 11789 %{
 11790   match(Set cr (CmpU op1 (LoadI op2)));
 11792   ins_cost(500); // XXX
 11793   format %{ "cmpl    $op1, $op2\t# unsigned" %}
 11794   opcode(0x3B); /* Opcode 3B /r */
 11795   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
 11796   ins_pipe(ialu_cr_reg_mem);
 11797 %}
 11799 // // // Cisc-spilled version of cmpU_rReg
 11800 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
 11801 // //%{
 11802 // //  match(Set cr (CmpU (LoadI op1) op2));
 11803 // //
 11804 // //  format %{ "CMPu   $op1,$op2" %}
 11805 // //  ins_cost(500);
 11806 // //  opcode(0x39);  /* Opcode 39 /r */
 11807 // //  ins_encode( OpcP, reg_mem( op1, op2) );
 11808 // //%}
 11810 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
 11811 %{
 11812   match(Set cr (CmpU src zero));
 11814   format %{ "testl  $src, $src\t# unsigned" %}
 11815   opcode(0x85);
 11816   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
 11817   ins_pipe(ialu_cr_reg_imm);
 11818 %}
 11820 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
 11821 %{
 11822   match(Set cr (CmpP op1 op2));
 11824   format %{ "cmpq    $op1, $op2\t# ptr" %}
 11825   opcode(0x3B); /* Opcode 3B /r */
 11826   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
 11827   ins_pipe(ialu_cr_reg_reg);
 11828 %}
 11830 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
 11831 %{
 11832   match(Set cr (CmpP op1 (LoadP op2)));
 11834   ins_cost(500); // XXX
 11835   format %{ "cmpq    $op1, $op2\t# ptr" %}
 11836   opcode(0x3B); /* Opcode 3B /r */
 11837   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
 11838   ins_pipe(ialu_cr_reg_mem);
 11839 %}
 11841 // // // Cisc-spilled version of cmpP_rReg
 11842 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
 11843 // //%{
 11844 // //  match(Set cr (CmpP (LoadP op1) op2));
 11845 // //
 11846 // //  format %{ "CMPu   $op1,$op2" %}
 11847 // //  ins_cost(500);
 11848 // //  opcode(0x39);  /* Opcode 39 /r */
 11849 // //  ins_encode( OpcP, reg_mem( op1, op2) );
 11850 // //%}
 11852 // XXX this is generalized by compP_rReg_mem???
 11853 // Compare raw pointer (used in out-of-heap check).
 11854 // Only works because non-oop pointers must be raw pointers
 11855 // and raw pointers have no anti-dependencies.
 11856 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
 11857 %{
 11858   predicate(!n->in(2)->in(2)->bottom_type()->isa_oop_ptr());
 11859   match(Set cr (CmpP op1 (LoadP op2)));
 11861   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
 11862   opcode(0x3B); /* Opcode 3B /r */
 11863   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
 11864   ins_pipe(ialu_cr_reg_mem);
 11865 %}
 11867 // This will generate a signed flags result. This should be OK since
 11868 // any compare to a zero should be eq/neq.
 11869 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
 11870 %{
 11871   match(Set cr (CmpP src zero));
 11873   format %{ "testq   $src, $src\t# ptr" %}
 11874   opcode(0x85);
 11875   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
 11876   ins_pipe(ialu_cr_reg_imm);
 11877 %}
 11879 // This will generate a signed flags result. This should be OK since
 11880 // any compare to a zero should be eq/neq.
 11881 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
 11882 %{
 11883   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
 11884   match(Set cr (CmpP (LoadP op) zero));
 11886   ins_cost(500); // XXX
 11887   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
 11888   opcode(0xF7); /* Opcode F7 /0 */
 11889   ins_encode(REX_mem_wide(op),
 11890              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
 11891   ins_pipe(ialu_cr_reg_imm);
 11892 %}
 11894 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
 11895 %{
 11896   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
 11897   match(Set cr (CmpP (LoadP mem) zero));
 11899   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
 11900   ins_encode %{
 11901     __ cmpq(r12, $mem$$Address);
 11902   %}
 11903   ins_pipe(ialu_cr_reg_mem);
 11904 %}
 11906 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
 11907 %{
 11908   match(Set cr (CmpN op1 op2));
 11910   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
 11911   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
 11912   ins_pipe(ialu_cr_reg_reg);
 11913 %}
 11915 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
 11916 %{
 11917   match(Set cr (CmpN src (LoadN mem)));
 11919   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
 11920   ins_encode %{
 11921     __ cmpl($src$$Register, $mem$$Address);
 11922   %}
 11923   ins_pipe(ialu_cr_reg_mem);
 11924 %}
 11926 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
 11927   match(Set cr (CmpN op1 op2));
 11929   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
 11930   ins_encode %{
 11931     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
 11932   %}
 11933   ins_pipe(ialu_cr_reg_imm);
 11934 %}
 11936 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
 11937 %{
 11938   match(Set cr (CmpN src (LoadN mem)));
 11940   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
 11941   ins_encode %{
 11942     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
 11943   %}
 11944   ins_pipe(ialu_cr_reg_mem);
 11945 %}
 11947 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
 11948   match(Set cr (CmpN src zero));
 11950   format %{ "testl   $src, $src\t# compressed ptr" %}
 11951   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
 11952   ins_pipe(ialu_cr_reg_imm);
 11953 %}
 11955 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
 11956 %{
 11957   predicate(Universe::narrow_oop_base() != NULL);
 11958   match(Set cr (CmpN (LoadN mem) zero));
 11960   ins_cost(500); // XXX
 11961   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
 11962   ins_encode %{
 11963     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
 11964   %}
 11965   ins_pipe(ialu_cr_reg_mem);
 11966 %}
 11968 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
 11969 %{
 11970   predicate(Universe::narrow_oop_base() == NULL);
 11971   match(Set cr (CmpN (LoadN mem) zero));
 11973   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
 11974   ins_encode %{
 11975     __ cmpl(r12, $mem$$Address);
 11976   %}
 11977   ins_pipe(ialu_cr_reg_mem);
 11978 %}
 11980 // Yanked all unsigned pointer compare operations.
 11981 // Pointer compares are done with CmpP which is already unsigned.
 11983 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
 11984 %{
 11985   match(Set cr (CmpL op1 op2));
 11987   format %{ "cmpq    $op1, $op2" %}
 11988   opcode(0x3B);  /* Opcode 3B /r */
 11989   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
 11990   ins_pipe(ialu_cr_reg_reg);
 11991 %}
 11993 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
 11994 %{
 11995   match(Set cr (CmpL op1 op2));
 11997   format %{ "cmpq    $op1, $op2" %}
 11998   opcode(0x81, 0x07); /* Opcode 81 /7 */
 11999   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
 12000   ins_pipe(ialu_cr_reg_imm);
 12001 %}
 12003 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
 12004 %{
 12005   match(Set cr (CmpL op1 (LoadL op2)));
 12007   format %{ "cmpq    $op1, $op2" %}
 12008   opcode(0x3B); /* Opcode 3B /r */
 12009   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
 12010   ins_pipe(ialu_cr_reg_mem);
 12011 %}
 12013 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
 12014 %{
 12015   match(Set cr (CmpL src zero));
 12017   format %{ "testq   $src, $src" %}
 12018   opcode(0x85);
 12019   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
 12020   ins_pipe(ialu_cr_reg_imm);
 12021 %}
 12023 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
 12024 %{
 12025   match(Set cr (CmpL (AndL src con) zero));
 12027   format %{ "testq   $src, $con\t# long" %}
 12028   opcode(0xF7, 0x00);
 12029   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
 12030   ins_pipe(ialu_cr_reg_imm);
 12031 %}
 12033 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
 12034 %{
 12035   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
 12037   format %{ "testq   $src, $mem" %}
 12038   opcode(0x85);
 12039   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
 12040   ins_pipe(ialu_cr_reg_mem);
 12041 %}
 12043 // Manifest a CmpL result in an integer register.  Very painful.
 12044 // This is the test to avoid.
 12045 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
 12046 %{
 12047   match(Set dst (CmpL3 src1 src2));
 12048   effect(KILL flags);
 12050   ins_cost(275); // XXX
 12051   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
 12052             "movl    $dst, -1\n\t"
 12053             "jl,s    done\n\t"
 12054             "setne   $dst\n\t"
 12055             "movzbl  $dst, $dst\n\t"
 12056     "done:" %}
 12057   ins_encode(cmpl3_flag(src1, src2, dst));
 12058   ins_pipe(pipe_slow);
 12059 %}
 12061 //----------Max and Min--------------------------------------------------------
 12062 // Min Instructions
 12064 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
 12065 %{
 12066   effect(USE_DEF dst, USE src, USE cr);
 12068   format %{ "cmovlgt $dst, $src\t# min" %}
 12069   opcode(0x0F, 0x4F);
 12070   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
 12071   ins_pipe(pipe_cmov_reg);
 12072 %}
 12075 instruct minI_rReg(rRegI dst, rRegI src)
 12076 %{
 12077   match(Set dst (MinI dst src));
 12079   ins_cost(200);
 12080   expand %{
 12081     rFlagsReg cr;
 12082     compI_rReg(cr, dst, src);
 12083     cmovI_reg_g(dst, src, cr);
 12084   %}
 12085 %}
 12087 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
 12088 %{
 12089   effect(USE_DEF dst, USE src, USE cr);
 12091   format %{ "cmovllt $dst, $src\t# max" %}
 12092   opcode(0x0F, 0x4C);
 12093   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
 12094   ins_pipe(pipe_cmov_reg);
 12095 %}
 12098 instruct maxI_rReg(rRegI dst, rRegI src)
 12099 %{
 12100   match(Set dst (MaxI dst src));
 12102   ins_cost(200);
 12103   expand %{
 12104     rFlagsReg cr;
 12105     compI_rReg(cr, dst, src);
 12106     cmovI_reg_l(dst, src, cr);
 12107   %}
 12108 %}
 12110 // ============================================================================
 12111 // Branch Instructions
 12113 // Jump Direct - Label defines a relative address from JMP+1
 12114 instruct jmpDir(label labl)
 12115 %{
 12116   match(Goto);
 12117   effect(USE labl);
 12119   ins_cost(300);
 12120   format %{ "jmp     $labl" %}
 12121   size(5);
 12122   opcode(0xE9);
 12123   ins_encode(OpcP, Lbl(labl));
 12124   ins_pipe(pipe_jmp);
 12125   ins_pc_relative(1);
 12126 %}
 12128 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 12129 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
 12130 %{
 12131   match(If cop cr);
 12132   effect(USE labl);
 12134   ins_cost(300);
 12135   format %{ "j$cop     $labl" %}
 12136   size(6);
 12137   opcode(0x0F, 0x80);
 12138   ins_encode(Jcc(cop, labl));
 12139   ins_pipe(pipe_jcc);
 12140   ins_pc_relative(1);
 12141 %}
 12143 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 12144 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
 12145 %{
 12146   match(CountedLoopEnd cop cr);
 12147   effect(USE labl);
 12149   ins_cost(300);
 12150   format %{ "j$cop     $labl\t# loop end" %}
 12151   size(6);
 12152   opcode(0x0F, 0x80);
 12153   ins_encode(Jcc(cop, labl));
 12154   ins_pipe(pipe_jcc);
 12155   ins_pc_relative(1);
 12156 %}
 12158 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 12159 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
 12160   match(CountedLoopEnd cop cmp);
 12161   effect(USE labl);
 12163   ins_cost(300);
 12164   format %{ "j$cop,u   $labl\t# loop end" %}
 12165   size(6);
 12166   opcode(0x0F, 0x80);
 12167   ins_encode(Jcc(cop, labl));
 12168   ins_pipe(pipe_jcc);
 12169   ins_pc_relative(1);
 12170 %}
 12172 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
 12173   match(CountedLoopEnd cop cmp);
 12174   effect(USE labl);
 12176   ins_cost(200);
 12177   format %{ "j$cop,u   $labl\t# loop end" %}
 12178   size(6);
 12179   opcode(0x0F, 0x80);
 12180   ins_encode(Jcc(cop, labl));
 12181   ins_pipe(pipe_jcc);
 12182   ins_pc_relative(1);
 12183 %}
 12185 // Jump Direct Conditional - using unsigned comparison
 12186 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
 12187   match(If cop cmp);
 12188   effect(USE labl);
 12190   ins_cost(300);
 12191   format %{ "j$cop,u  $labl" %}
 12192   size(6);
 12193   opcode(0x0F, 0x80);
 12194   ins_encode(Jcc(cop, labl));
 12195   ins_pipe(pipe_jcc);
 12196   ins_pc_relative(1);
 12197 %}
 12199 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
 12200   match(If cop cmp);
 12201   effect(USE labl);
 12203   ins_cost(200);
 12204   format %{ "j$cop,u  $labl" %}
 12205   size(6);
 12206   opcode(0x0F, 0x80);
 12207   ins_encode(Jcc(cop, labl));
 12208   ins_pipe(pipe_jcc);
 12209   ins_pc_relative(1);
 12210 %}
 12212 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
 12213   match(If cop cmp);
 12214   effect(USE labl);
 12216   ins_cost(200);
 12217   format %{ $$template
 12218     if ($cop$$cmpcode == Assembler::notEqual) {
 12219       $$emit$$"jp,u   $labl\n\t"
 12220       $$emit$$"j$cop,u   $labl"
 12221     } else {
 12222       $$emit$$"jp,u   done\n\t"
 12223       $$emit$$"j$cop,u   $labl\n\t"
 12224       $$emit$$"done:"
 12226   %}
 12227   size(12);
 12228   opcode(0x0F, 0x80);
 12229   ins_encode %{
 12230     Label* l = $labl$$label;
 12231     $$$emit8$primary;
 12232     emit_cc(cbuf, $secondary, Assembler::parity);
 12233     int parity_disp = -1;
 12234     if ($cop$$cmpcode == Assembler::notEqual) {
 12235        // the two jumps 6 bytes apart so the jump distances are too
 12236        parity_disp = l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0;
 12237     } else if ($cop$$cmpcode == Assembler::equal) {
 12238        parity_disp = 6;
 12239     } else {
 12240        ShouldNotReachHere();
 12242     emit_d32(cbuf, parity_disp);
 12243     $$$emit8$primary;
 12244     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 12245     int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0;
 12246     emit_d32(cbuf, disp);
 12247   %}
 12248   ins_pipe(pipe_jcc);
 12249   ins_pc_relative(1);
 12250 %}
 12252 // ============================================================================
 12253 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
 12254 // superklass array for an instance of the superklass.  Set a hidden
 12255 // internal cache on a hit (cache is checked with exposed code in
 12256 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
 12257 // encoding ALSO sets flags.
 12259 instruct partialSubtypeCheck(rdi_RegP result,
 12260                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
 12261                              rFlagsReg cr)
 12262 %{
 12263   match(Set result (PartialSubtypeCheck sub super));
 12264   effect(KILL rcx, KILL cr);
 12266   ins_cost(1100);  // slightly larger than the next version
 12267   format %{ "movq    rdi, [$sub + (sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes())]\n\t"
 12268             "movl    rcx, [rdi + arrayOopDesc::length_offset_in_bytes()]\t# length to scan\n\t"
 12269             "addq    rdi, arrayOopDex::base_offset_in_bytes(T_OBJECT)\t# Skip to start of data; set NZ in case count is zero\n\t"
 12270             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
 12271             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
 12272             "movq    [$sub + (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes())], $super\t# Hit: update cache\n\t"
 12273             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
 12274     "miss:\t" %}
 12276   opcode(0x1); // Force a XOR of RDI
 12277   ins_encode(enc_PartialSubtypeCheck());
 12278   ins_pipe(pipe_slow);
 12279 %}
 12281 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
 12282                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
 12283                                      immP0 zero,
 12284                                      rdi_RegP result)
 12285 %{
 12286   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
 12287   effect(KILL rcx, KILL result);
 12289   ins_cost(1000);
 12290   format %{ "movq    rdi, [$sub + (sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes())]\n\t"
 12291             "movl    rcx, [rdi + arrayOopDesc::length_offset_in_bytes()]\t# length to scan\n\t"
 12292             "addq    rdi, arrayOopDex::base_offset_in_bytes(T_OBJECT)\t# Skip to start of data; set NZ in case count is zero\n\t"
 12293             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
 12294             "jne,s   miss\t\t# Missed: flags nz\n\t"
 12295             "movq    [$sub + (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes())], $super\t# Hit: update cache\n\t"
 12296     "miss:\t" %}
 12298   opcode(0x0); // No need to XOR RDI
 12299   ins_encode(enc_PartialSubtypeCheck());
 12300   ins_pipe(pipe_slow);
 12301 %}
 12303 // ============================================================================
 12304 // Branch Instructions -- short offset versions
 12305 //
 12306 // These instructions are used to replace jumps of a long offset (the default
 12307 // match) with jumps of a shorter offset.  These instructions are all tagged
 12308 // with the ins_short_branch attribute, which causes the ADLC to suppress the
 12309 // match rules in general matching.  Instead, the ADLC generates a conversion
 12310 // method in the MachNode which can be used to do in-place replacement of the
 12311 // long variant with the shorter variant.  The compiler will determine if a
 12312 // branch can be taken by the is_short_branch_offset() predicate in the machine
 12313 // specific code section of the file.
 12315 // Jump Direct - Label defines a relative address from JMP+1
 12316 instruct jmpDir_short(label labl) %{
 12317   match(Goto);
 12318   effect(USE labl);
 12320   ins_cost(300);
 12321   format %{ "jmp,s   $labl" %}
 12322   size(2);
 12323   opcode(0xEB);
 12324   ins_encode(OpcP, LblShort(labl));
 12325   ins_pipe(pipe_jmp);
 12326   ins_pc_relative(1);
 12327   ins_short_branch(1);
 12328 %}
 12330 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 12331 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
 12332   match(If cop cr);
 12333   effect(USE labl);
 12335   ins_cost(300);
 12336   format %{ "j$cop,s   $labl" %}
 12337   size(2);
 12338   opcode(0x70);
 12339   ins_encode(JccShort(cop, labl));
 12340   ins_pipe(pipe_jcc);
 12341   ins_pc_relative(1);
 12342   ins_short_branch(1);
 12343 %}
 12345 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 12346 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
 12347   match(CountedLoopEnd cop cr);
 12348   effect(USE labl);
 12350   ins_cost(300);
 12351   format %{ "j$cop,s   $labl\t# loop end" %}
 12352   size(2);
 12353   opcode(0x70);
 12354   ins_encode(JccShort(cop, labl));
 12355   ins_pipe(pipe_jcc);
 12356   ins_pc_relative(1);
 12357   ins_short_branch(1);
 12358 %}
 12360 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 12361 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
 12362   match(CountedLoopEnd cop cmp);
 12363   effect(USE labl);
 12365   ins_cost(300);
 12366   format %{ "j$cop,us  $labl\t# loop end" %}
 12367   size(2);
 12368   opcode(0x70);
 12369   ins_encode(JccShort(cop, labl));
 12370   ins_pipe(pipe_jcc);
 12371   ins_pc_relative(1);
 12372   ins_short_branch(1);
 12373 %}
 12375 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
 12376   match(CountedLoopEnd cop cmp);
 12377   effect(USE labl);
 12379   ins_cost(300);
 12380   format %{ "j$cop,us  $labl\t# loop end" %}
 12381   size(2);
 12382   opcode(0x70);
 12383   ins_encode(JccShort(cop, labl));
 12384   ins_pipe(pipe_jcc);
 12385   ins_pc_relative(1);
 12386   ins_short_branch(1);
 12387 %}
 12389 // Jump Direct Conditional - using unsigned comparison
 12390 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
 12391   match(If cop cmp);
 12392   effect(USE labl);
 12394   ins_cost(300);
 12395   format %{ "j$cop,us  $labl" %}
 12396   size(2);
 12397   opcode(0x70);
 12398   ins_encode(JccShort(cop, labl));
 12399   ins_pipe(pipe_jcc);
 12400   ins_pc_relative(1);
 12401   ins_short_branch(1);
 12402 %}
 12404 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
 12405   match(If cop cmp);
 12406   effect(USE labl);
 12408   ins_cost(300);
 12409   format %{ "j$cop,us  $labl" %}
 12410   size(2);
 12411   opcode(0x70);
 12412   ins_encode(JccShort(cop, labl));
 12413   ins_pipe(pipe_jcc);
 12414   ins_pc_relative(1);
 12415   ins_short_branch(1);
 12416 %}
 12418 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
 12419   match(If cop cmp);
 12420   effect(USE labl);
 12422   ins_cost(300);
 12423   format %{ $$template
 12424     if ($cop$$cmpcode == Assembler::notEqual) {
 12425       $$emit$$"jp,u,s   $labl\n\t"
 12426       $$emit$$"j$cop,u,s   $labl"
 12427     } else {
 12428       $$emit$$"jp,u,s   done\n\t"
 12429       $$emit$$"j$cop,u,s  $labl\n\t"
 12430       $$emit$$"done:"
 12432   %}
 12433   size(4);
 12434   opcode(0x70);
 12435   ins_encode %{
 12436     Label* l = $labl$$label;
 12437     emit_cc(cbuf, $primary, Assembler::parity);
 12438     int parity_disp = -1;
 12439     if ($cop$$cmpcode == Assembler::notEqual) {
 12440       parity_disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
 12441     } else if ($cop$$cmpcode == Assembler::equal) {
 12442       parity_disp = 2;
 12443     } else {
 12444       ShouldNotReachHere();
 12446     emit_d8(cbuf, parity_disp);
 12447     emit_cc(cbuf, $primary, $cop$$cmpcode);
 12448     int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
 12449     emit_d8(cbuf, disp);
 12450     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
 12451     assert(-128 <= parity_disp && parity_disp <= 127, "Displacement too large for short jmp");
 12452   %}
 12453   ins_pipe(pipe_jcc);
 12454   ins_pc_relative(1);
 12455   ins_short_branch(1);
 12456 %}
 12458 // ============================================================================
 12459 // inlined locking and unlocking
 12461 instruct cmpFastLock(rFlagsReg cr,
 12462                      rRegP object, rRegP box, rax_RegI tmp, rRegP scr)
 12463 %{
 12464   match(Set cr (FastLock object box));
 12465   effect(TEMP tmp, TEMP scr);
 12467   ins_cost(300);
 12468   format %{ "fastlock $object,$box,$tmp,$scr" %}
 12469   ins_encode(Fast_Lock(object, box, tmp, scr));
 12470   ins_pipe(pipe_slow);
 12471   ins_pc_relative(1);
 12472 %}
 12474 instruct cmpFastUnlock(rFlagsReg cr,
 12475                        rRegP object, rax_RegP box, rRegP tmp)
 12476 %{
 12477   match(Set cr (FastUnlock object box));
 12478   effect(TEMP tmp);
 12480   ins_cost(300);
 12481   format %{ "fastunlock $object, $box, $tmp" %}
 12482   ins_encode(Fast_Unlock(object, box, tmp));
 12483   ins_pipe(pipe_slow);
 12484   ins_pc_relative(1);
 12485 %}
 12488 // ============================================================================
 12489 // Safepoint Instructions
 12490 instruct safePoint_poll(rFlagsReg cr)
 12491 %{
 12492   match(SafePoint);
 12493   effect(KILL cr);
 12495   format %{ "testl   rax, [rip + #offset_to_poll_page]\t"
 12496             "# Safepoint: poll for GC" %}
 12497   size(6); // Opcode + ModRM + Disp32 == 6 bytes
 12498   ins_cost(125);
 12499   ins_encode(enc_safepoint_poll);
 12500   ins_pipe(ialu_reg_mem);
 12501 %}
 12503 // ============================================================================
 12504 // Procedure Call/Return Instructions
 12505 // Call Java Static Instruction
 12506 // Note: If this code changes, the corresponding ret_addr_offset() and
 12507 //       compute_padding() functions will have to be adjusted.
 12508 instruct CallStaticJavaDirect(method meth) %{
 12509   match(CallStaticJava);
 12510   predicate(!((CallStaticJavaNode*) n)->is_method_handle_invoke());
 12511   effect(USE meth);
 12513   ins_cost(300);
 12514   format %{ "call,static " %}
 12515   opcode(0xE8); /* E8 cd */
 12516   ins_encode(Java_Static_Call(meth), call_epilog);
 12517   ins_pipe(pipe_slow);
 12518   ins_pc_relative(1);
 12519   ins_alignment(4);
 12520 %}
 12522 // Call Java Static Instruction (method handle version)
 12523 // Note: If this code changes, the corresponding ret_addr_offset() and
 12524 //       compute_padding() functions will have to be adjusted.
 12525 instruct CallStaticJavaHandle(method meth, rbp_RegP rbp_mh_SP_save) %{
 12526   match(CallStaticJava);
 12527   predicate(((CallStaticJavaNode*) n)->is_method_handle_invoke());
 12528   effect(USE meth);
 12529   // RBP is saved by all callees (for interpreter stack correction).
 12530   // We use it here for a similar purpose, in {preserve,restore}_SP.
 12532   ins_cost(300);
 12533   format %{ "call,static/MethodHandle " %}
 12534   opcode(0xE8); /* E8 cd */
 12535   ins_encode(preserve_SP,
 12536              Java_Static_Call(meth),
 12537              restore_SP,
 12538              call_epilog);
 12539   ins_pipe(pipe_slow);
 12540   ins_pc_relative(1);
 12541   ins_alignment(4);
 12542 %}
 12544 // Call Java Dynamic Instruction
 12545 // Note: If this code changes, the corresponding ret_addr_offset() and
 12546 //       compute_padding() functions will have to be adjusted.
 12547 instruct CallDynamicJavaDirect(method meth)
 12548 %{
 12549   match(CallDynamicJava);
 12550   effect(USE meth);
 12552   ins_cost(300);
 12553   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
 12554             "call,dynamic " %}
 12555   opcode(0xE8); /* E8 cd */
 12556   ins_encode(Java_Dynamic_Call(meth), call_epilog);
 12557   ins_pipe(pipe_slow);
 12558   ins_pc_relative(1);
 12559   ins_alignment(4);
 12560 %}
 12562 // Call Runtime Instruction
 12563 instruct CallRuntimeDirect(method meth)
 12564 %{
 12565   match(CallRuntime);
 12566   effect(USE meth);
 12568   ins_cost(300);
 12569   format %{ "call,runtime " %}
 12570   opcode(0xE8); /* E8 cd */
 12571   ins_encode(Java_To_Runtime(meth));
 12572   ins_pipe(pipe_slow);
 12573   ins_pc_relative(1);
 12574 %}
 12576 // Call runtime without safepoint
 12577 instruct CallLeafDirect(method meth)
 12578 %{
 12579   match(CallLeaf);
 12580   effect(USE meth);
 12582   ins_cost(300);
 12583   format %{ "call_leaf,runtime " %}
 12584   opcode(0xE8); /* E8 cd */
 12585   ins_encode(Java_To_Runtime(meth));
 12586   ins_pipe(pipe_slow);
 12587   ins_pc_relative(1);
 12588 %}
 12590 // Call runtime without safepoint
 12591 instruct CallLeafNoFPDirect(method meth)
 12592 %{
 12593   match(CallLeafNoFP);
 12594   effect(USE meth);
 12596   ins_cost(300);
 12597   format %{ "call_leaf_nofp,runtime " %}
 12598   opcode(0xE8); /* E8 cd */
 12599   ins_encode(Java_To_Runtime(meth));
 12600   ins_pipe(pipe_slow);
 12601   ins_pc_relative(1);
 12602 %}
 12604 // Return Instruction
 12605 // Remove the return address & jump to it.
 12606 // Notice: We always emit a nop after a ret to make sure there is room
 12607 // for safepoint patching
 12608 instruct Ret()
 12609 %{
 12610   match(Return);
 12612   format %{ "ret" %}
 12613   opcode(0xC3);
 12614   ins_encode(OpcP);
 12615   ins_pipe(pipe_jmp);
 12616 %}
 12618 // Tail Call; Jump from runtime stub to Java code.
 12619 // Also known as an 'interprocedural jump'.
 12620 // Target of jump will eventually return to caller.
 12621 // TailJump below removes the return address.
 12622 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
 12623 %{
 12624   match(TailCall jump_target method_oop);
 12626   ins_cost(300);
 12627   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
 12628   opcode(0xFF, 0x4); /* Opcode FF /4 */
 12629   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
 12630   ins_pipe(pipe_jmp);
 12631 %}
 12633 // Tail Jump; remove the return address; jump to target.
 12634 // TailCall above leaves the return address around.
 12635 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
 12636 %{
 12637   match(TailJump jump_target ex_oop);
 12639   ins_cost(300);
 12640   format %{ "popq    rdx\t# pop return address\n\t"
 12641             "jmp     $jump_target" %}
 12642   opcode(0xFF, 0x4); /* Opcode FF /4 */
 12643   ins_encode(Opcode(0x5a), // popq rdx
 12644              REX_reg(jump_target), OpcP, reg_opc(jump_target));
 12645   ins_pipe(pipe_jmp);
 12646 %}
 12648 // Create exception oop: created by stack-crawling runtime code.
 12649 // Created exception is now available to this handler, and is setup
 12650 // just prior to jumping to this handler.  No code emitted.
 12651 instruct CreateException(rax_RegP ex_oop)
 12652 %{
 12653   match(Set ex_oop (CreateEx));
 12655   size(0);
 12656   // use the following format syntax
 12657   format %{ "# exception oop is in rax; no code emitted" %}
 12658   ins_encode();
 12659   ins_pipe(empty);
 12660 %}
 12662 // Rethrow exception:
 12663 // The exception oop will come in the first argument position.
 12664 // Then JUMP (not call) to the rethrow stub code.
 12665 instruct RethrowException()
 12666 %{
 12667   match(Rethrow);
 12669   // use the following format syntax
 12670   format %{ "jmp     rethrow_stub" %}
 12671   ins_encode(enc_rethrow);
 12672   ins_pipe(pipe_jmp);
 12673 %}
 12676 //----------PEEPHOLE RULES-----------------------------------------------------
 12677 // These must follow all instruction definitions as they use the names
 12678 // defined in the instructions definitions.
 12679 //
 12680 // peepmatch ( root_instr_name [preceding_instruction]* );
 12681 //
 12682 // peepconstraint %{
 12683 // (instruction_number.operand_name relational_op instruction_number.operand_name
 12684 //  [, ...] );
 12685 // // instruction numbers are zero-based using left to right order in peepmatch
 12686 //
 12687 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
 12688 // // provide an instruction_number.operand_name for each operand that appears
 12689 // // in the replacement instruction's match rule
 12690 //
 12691 // ---------VM FLAGS---------------------------------------------------------
 12692 //
 12693 // All peephole optimizations can be turned off using -XX:-OptoPeephole
 12694 //
 12695 // Each peephole rule is given an identifying number starting with zero and
 12696 // increasing by one in the order seen by the parser.  An individual peephole
 12697 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
 12698 // on the command-line.
 12699 //
 12700 // ---------CURRENT LIMITATIONS----------------------------------------------
 12701 //
 12702 // Only match adjacent instructions in same basic block
 12703 // Only equality constraints
 12704 // Only constraints between operands, not (0.dest_reg == RAX_enc)
 12705 // Only one replacement instruction
 12706 //
 12707 // ---------EXAMPLE----------------------------------------------------------
 12708 //
 12709 // // pertinent parts of existing instructions in architecture description
 12710 // instruct movI(rRegI dst, rRegI src)
 12711 // %{
 12712 //   match(Set dst (CopyI src));
 12713 // %}
 12714 //
 12715 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
 12716 // %{
 12717 //   match(Set dst (AddI dst src));
 12718 //   effect(KILL cr);
 12719 // %}
 12720 //
 12721 // // Change (inc mov) to lea
 12722 // peephole %{
 12723 //   // increment preceeded by register-register move
 12724 //   peepmatch ( incI_rReg movI );
 12725 //   // require that the destination register of the increment
 12726 //   // match the destination register of the move
 12727 //   peepconstraint ( 0.dst == 1.dst );
 12728 //   // construct a replacement instruction that sets
 12729 //   // the destination to ( move's source register + one )
 12730 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
 12731 // %}
 12732 //
 12734 // Implementation no longer uses movX instructions since
 12735 // machine-independent system no longer uses CopyX nodes.
 12736 //
 12737 // peephole
 12738 // %{
 12739 //   peepmatch (incI_rReg movI);
 12740 //   peepconstraint (0.dst == 1.dst);
 12741 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
 12742 // %}
 12744 // peephole
 12745 // %{
 12746 //   peepmatch (decI_rReg movI);
 12747 //   peepconstraint (0.dst == 1.dst);
 12748 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
 12749 // %}
 12751 // peephole
 12752 // %{
 12753 //   peepmatch (addI_rReg_imm movI);
 12754 //   peepconstraint (0.dst == 1.dst);
 12755 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
 12756 // %}
 12758 // peephole
 12759 // %{
 12760 //   peepmatch (incL_rReg movL);
 12761 //   peepconstraint (0.dst == 1.dst);
 12762 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
 12763 // %}
 12765 // peephole
 12766 // %{
 12767 //   peepmatch (decL_rReg movL);
 12768 //   peepconstraint (0.dst == 1.dst);
 12769 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
 12770 // %}
 12772 // peephole
 12773 // %{
 12774 //   peepmatch (addL_rReg_imm movL);
 12775 //   peepconstraint (0.dst == 1.dst);
 12776 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
 12777 // %}
 12779 // peephole
 12780 // %{
 12781 //   peepmatch (addP_rReg_imm movP);
 12782 //   peepconstraint (0.dst == 1.dst);
 12783 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
 12784 // %}
 12786 // // Change load of spilled value to only a spill
 12787 // instruct storeI(memory mem, rRegI src)
 12788 // %{
 12789 //   match(Set mem (StoreI mem src));
 12790 // %}
 12791 //
 12792 // instruct loadI(rRegI dst, memory mem)
 12793 // %{
 12794 //   match(Set dst (LoadI mem));
 12795 // %}
 12796 //
 12798 peephole
 12799 %{
 12800   peepmatch (loadI storeI);
 12801   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
 12802   peepreplace (storeI(1.mem 1.mem 1.src));
 12803 %}
 12805 peephole
 12806 %{
 12807   peepmatch (loadL storeL);
 12808   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
 12809   peepreplace (storeL(1.mem 1.mem 1.src));
 12810 %}
 12812 //----------SMARTSPILL RULES---------------------------------------------------
 12813 // These must follow all instruction definitions as they use the names
 12814 // defined in the instructions definitions.

mercurial