src/cpu/x86/vm/x86_32.ad

Mon, 11 Apr 2011 15:30:31 -0700

author
kvn
date
Mon, 11 Apr 2011 15:30:31 -0700
changeset 2761
15c9a0e16269
parent 2683
7e88bdae86ec
child 2877
bad7ecd0b6ed
permissions
-rw-r--r--

7035713: 3DNow Prefetch Instruction Support
Summary: The upcoming processors from AMD are the first that support 3dnow prefetch without supporting the 3dnow instruction set.
Reviewed-by: kvn
Contributed-by: tom.deneau@amd.com

     1 //
     2 // Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
     3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     4 //
     5 // This code is free software; you can redistribute it and/or modify it
     6 // under the terms of the GNU General Public License version 2 only, as
     7 // published by the Free Software Foundation.
     8 //
     9 // This code is distributed in the hope that it will be useful, but WITHOUT
    10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    12 // version 2 for more details (a copy is included in the LICENSE file that
    13 // accompanied this code).
    14 //
    15 // You should have received a copy of the GNU General Public License version
    16 // 2 along with this work; if not, write to the Free Software Foundation,
    17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
    18 //
    19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
    20 // or visit www.oracle.com if you need additional information or have any
    21 // questions.
    22 //
    23 //
    25 // X86 Architecture Description File
    27 //----------REGISTER DEFINITION BLOCK------------------------------------------
    28 // This information is used by the matcher and the register allocator to
    29 // describe individual registers and classes of registers within the target
    30 // archtecture.
    32 register %{
    33 //----------Architecture Description Register Definitions----------------------
    34 // General Registers
    35 // "reg_def"  name ( register save type, C convention save type,
    36 //                   ideal register type, encoding );
    37 // Register Save Types:
    38 //
    39 // NS  = No-Save:       The register allocator assumes that these registers
    40 //                      can be used without saving upon entry to the method, &
    41 //                      that they do not need to be saved at call sites.
    42 //
    43 // SOC = Save-On-Call:  The register allocator assumes that these registers
    44 //                      can be used without saving upon entry to the method,
    45 //                      but that they must be saved at call sites.
    46 //
    47 // SOE = Save-On-Entry: The register allocator assumes that these registers
    48 //                      must be saved before using them upon entry to the
    49 //                      method, but they do not need to be saved at call
    50 //                      sites.
    51 //
    52 // AS  = Always-Save:   The register allocator assumes that these registers
    53 //                      must be saved before using them upon entry to the
    54 //                      method, & that they must be saved at call sites.
    55 //
    56 // Ideal Register Type is used to determine how to save & restore a
    57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
    58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
    59 //
    60 // The encoding number is the actual bit-pattern placed into the opcodes.
    62 // General Registers
    63 // Previously set EBX, ESI, and EDI as save-on-entry for java code
    64 // Turn off SOE in java-code due to frequent use of uncommon-traps.
    65 // Now that allocator is better, turn on ESI and EDI as SOE registers.
    67 reg_def EBX(SOC, SOE, Op_RegI, 3, rbx->as_VMReg());
    68 reg_def ECX(SOC, SOC, Op_RegI, 1, rcx->as_VMReg());
    69 reg_def ESI(SOC, SOE, Op_RegI, 6, rsi->as_VMReg());
    70 reg_def EDI(SOC, SOE, Op_RegI, 7, rdi->as_VMReg());
    71 // now that adapter frames are gone EBP is always saved and restored by the prolog/epilog code
    72 reg_def EBP(NS, SOE, Op_RegI, 5, rbp->as_VMReg());
    73 reg_def EDX(SOC, SOC, Op_RegI, 2, rdx->as_VMReg());
    74 reg_def EAX(SOC, SOC, Op_RegI, 0, rax->as_VMReg());
    75 reg_def ESP( NS,  NS, Op_RegI, 4, rsp->as_VMReg());
    77 // Special Registers
    78 reg_def EFLAGS(SOC, SOC, 0, 8, VMRegImpl::Bad());
    80 // Float registers.  We treat TOS/FPR0 special.  It is invisible to the
    81 // allocator, and only shows up in the encodings.
    82 reg_def FPR0L( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());
    83 reg_def FPR0H( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());
    84 // Ok so here's the trick FPR1 is really st(0) except in the midst
    85 // of emission of assembly for a machnode. During the emission the fpu stack
    86 // is pushed making FPR1 == st(1) temporarily. However at any safepoint
    87 // the stack will not have this element so FPR1 == st(0) from the
    88 // oopMap viewpoint. This same weirdness with numbering causes
    89 // instruction encoding to have to play games with the register
    90 // encode to correct for this 0/1 issue. See MachSpillCopyNode::implementation
    91 // where it does flt->flt moves to see an example
    92 //
    93 reg_def FPR1L( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg());
    94 reg_def FPR1H( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg()->next());
    95 reg_def FPR2L( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg());
    96 reg_def FPR2H( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg()->next());
    97 reg_def FPR3L( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg());
    98 reg_def FPR3H( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg()->next());
    99 reg_def FPR4L( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg());
   100 reg_def FPR4H( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg()->next());
   101 reg_def FPR5L( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg());
   102 reg_def FPR5H( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg()->next());
   103 reg_def FPR6L( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg());
   104 reg_def FPR6H( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg()->next());
   105 reg_def FPR7L( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg());
   106 reg_def FPR7H( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg()->next());
   108 // XMM registers.  128-bit registers or 4 words each, labeled a-d.
   109 // Word a in each register holds a Float, words ab hold a Double.
   110 // We currently do not use the SIMD capabilities, so registers cd
   111 // are unused at the moment.
   112 reg_def XMM0a( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());
   113 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next());
   114 reg_def XMM1a( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());
   115 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next());
   116 reg_def XMM2a( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());
   117 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next());
   118 reg_def XMM3a( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());
   119 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next());
   120 reg_def XMM4a( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());
   121 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next());
   122 reg_def XMM5a( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());
   123 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next());
   124 reg_def XMM6a( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());
   125 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next());
   126 reg_def XMM7a( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());
   127 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next());
   129 // Specify priority of register selection within phases of register
   130 // allocation.  Highest priority is first.  A useful heuristic is to
   131 // give registers a low priority when they are required by machine
   132 // instructions, like EAX and EDX.  Registers which are used as
   133 // pairs must fall on an even boundary (witness the FPR#L's in this list).
   134 // For the Intel integer registers, the equivalent Long pairs are
   135 // EDX:EAX, EBX:ECX, and EDI:EBP.
   136 alloc_class chunk0( ECX,   EBX,   EBP,   EDI,   EAX,   EDX,   ESI, ESP,
   137                     FPR0L, FPR0H, FPR1L, FPR1H, FPR2L, FPR2H,
   138                     FPR3L, FPR3H, FPR4L, FPR4H, FPR5L, FPR5H,
   139                     FPR6L, FPR6H, FPR7L, FPR7H );
   141 alloc_class chunk1( XMM0a, XMM0b,
   142                     XMM1a, XMM1b,
   143                     XMM2a, XMM2b,
   144                     XMM3a, XMM3b,
   145                     XMM4a, XMM4b,
   146                     XMM5a, XMM5b,
   147                     XMM6a, XMM6b,
   148                     XMM7a, XMM7b, EFLAGS);
   151 //----------Architecture Description Register Classes--------------------------
   152 // Several register classes are automatically defined based upon information in
   153 // this architecture description.
   154 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
   155 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
   156 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
   157 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
   158 //
   159 // Class for all registers
   160 reg_class any_reg(EAX, EDX, EBP, EDI, ESI, ECX, EBX, ESP);
   161 // Class for general registers
   162 reg_class e_reg(EAX, EDX, EBP, EDI, ESI, ECX, EBX);
   163 // Class for general registers which may be used for implicit null checks on win95
   164 // Also safe for use by tailjump. We don't want to allocate in rbp,
   165 reg_class e_reg_no_rbp(EAX, EDX, EDI, ESI, ECX, EBX);
   166 // Class of "X" registers
   167 reg_class x_reg(EBX, ECX, EDX, EAX);
   168 // Class of registers that can appear in an address with no offset.
   169 // EBP and ESP require an extra instruction byte for zero offset.
   170 // Used in fast-unlock
   171 reg_class p_reg(EDX, EDI, ESI, EBX);
   172 // Class for general registers not including ECX
   173 reg_class ncx_reg(EAX, EDX, EBP, EDI, ESI, EBX);
   174 // Class for general registers not including EAX
   175 reg_class nax_reg(EDX, EDI, ESI, ECX, EBX);
   176 // Class for general registers not including EAX or EBX.
   177 reg_class nabx_reg(EDX, EDI, ESI, ECX, EBP);
   178 // Class of EAX (for multiply and divide operations)
   179 reg_class eax_reg(EAX);
   180 // Class of EBX (for atomic add)
   181 reg_class ebx_reg(EBX);
   182 // Class of ECX (for shift and JCXZ operations and cmpLTMask)
   183 reg_class ecx_reg(ECX);
   184 // Class of EDX (for multiply and divide operations)
   185 reg_class edx_reg(EDX);
   186 // Class of EDI (for synchronization)
   187 reg_class edi_reg(EDI);
   188 // Class of ESI (for synchronization)
   189 reg_class esi_reg(ESI);
   190 // Singleton class for interpreter's stack pointer
   191 reg_class ebp_reg(EBP);
   192 // Singleton class for stack pointer
   193 reg_class sp_reg(ESP);
   194 // Singleton class for instruction pointer
   195 // reg_class ip_reg(EIP);
   196 // Singleton class for condition codes
   197 reg_class int_flags(EFLAGS);
   198 // Class of integer register pairs
   199 reg_class long_reg( EAX,EDX, ECX,EBX, EBP,EDI );
   200 // Class of integer register pairs that aligns with calling convention
   201 reg_class eadx_reg( EAX,EDX );
   202 reg_class ebcx_reg( ECX,EBX );
   203 // Not AX or DX, used in divides
   204 reg_class nadx_reg( EBX,ECX,ESI,EDI,EBP );
   206 // Floating point registers.  Notice FPR0 is not a choice.
   207 // FPR0 is not ever allocated; we use clever encodings to fake
   208 // a 2-address instructions out of Intels FP stack.
   209 reg_class flt_reg( FPR1L,FPR2L,FPR3L,FPR4L,FPR5L,FPR6L,FPR7L );
   211 // make a register class for SSE registers
   212 reg_class xmm_reg(XMM0a, XMM1a, XMM2a, XMM3a, XMM4a, XMM5a, XMM6a, XMM7a);
   214 // make a double register class for SSE2 registers
   215 reg_class xdb_reg(XMM0a,XMM0b, XMM1a,XMM1b, XMM2a,XMM2b, XMM3a,XMM3b,
   216                   XMM4a,XMM4b, XMM5a,XMM5b, XMM6a,XMM6b, XMM7a,XMM7b );
   218 reg_class dbl_reg( FPR1L,FPR1H, FPR2L,FPR2H, FPR3L,FPR3H,
   219                    FPR4L,FPR4H, FPR5L,FPR5H, FPR6L,FPR6H,
   220                    FPR7L,FPR7H );
   222 reg_class flt_reg0( FPR1L );
   223 reg_class dbl_reg0( FPR1L,FPR1H );
   224 reg_class dbl_reg1( FPR2L,FPR2H );
   225 reg_class dbl_notreg0( FPR2L,FPR2H, FPR3L,FPR3H, FPR4L,FPR4H,
   226                        FPR5L,FPR5H, FPR6L,FPR6H, FPR7L,FPR7H );
   228 // XMM6 and XMM7 could be used as temporary registers for long, float and
   229 // double values for SSE2.
   230 reg_class xdb_reg6( XMM6a,XMM6b );
   231 reg_class xdb_reg7( XMM7a,XMM7b );
   232 %}
   235 //----------SOURCE BLOCK-------------------------------------------------------
   236 // This is a block of C++ code which provides values, functions, and
   237 // definitions necessary in the rest of the architecture description
   238 source_hpp %{
   239 // Must be visible to the DFA in dfa_x86_32.cpp
   240 extern bool is_operand_hi32_zero(Node* n);
   241 %}
   243 source %{
   244 #define   RELOC_IMM32    Assembler::imm_operand
   245 #define   RELOC_DISP32   Assembler::disp32_operand
   247 #define __ _masm.
   249 // How to find the high register of a Long pair, given the low register
   250 #define   HIGH_FROM_LOW(x) ((x)+2)
   252 // These masks are used to provide 128-bit aligned bitmasks to the XMM
   253 // instructions, to allow sign-masking or sign-bit flipping.  They allow
   254 // fast versions of NegF/NegD and AbsF/AbsD.
   256 // Note: 'double' and 'long long' have 32-bits alignment on x86.
   257 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
   258   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
   259   // of 128-bits operands for SSE instructions.
   260   jlong *operand = (jlong*)(((uintptr_t)adr)&((uintptr_t)(~0xF)));
   261   // Store the value to a 128-bits operand.
   262   operand[0] = lo;
   263   operand[1] = hi;
   264   return operand;
   265 }
   267 // Buffer for 128-bits masks used by SSE instructions.
   268 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
   270 // Static initialization during VM startup.
   271 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
   272 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
   273 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], CONST64(0x8000000080000000), CONST64(0x8000000080000000));
   274 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], CONST64(0x8000000000000000), CONST64(0x8000000000000000));
   276 // Offset hacking within calls.
   277 static int pre_call_FPU_size() {
   278   if (Compile::current()->in_24_bit_fp_mode())
   279     return 6; // fldcw
   280   return 0;
   281 }
   283 static int preserve_SP_size() {
   284   return LP64_ONLY(1 +) 2;  // [rex,] op, rm(reg/reg)
   285 }
   287 // !!!!! Special hack to get all type of calls to specify the byte offset
   288 //       from the start of the call to the point where the return address
   289 //       will point.
   290 int MachCallStaticJavaNode::ret_addr_offset() {
   291   int offset = 5 + pre_call_FPU_size();  // 5 bytes from start of call to where return address points
   292   if (_method_handle_invoke)
   293     offset += preserve_SP_size();
   294   return offset;
   295 }
   297 int MachCallDynamicJavaNode::ret_addr_offset() {
   298   return 10 + pre_call_FPU_size();  // 10 bytes from start of call to where return address points
   299 }
   301 static int sizeof_FFree_Float_Stack_All = -1;
   303 int MachCallRuntimeNode::ret_addr_offset() {
   304   assert(sizeof_FFree_Float_Stack_All != -1, "must have been emitted already");
   305   return sizeof_FFree_Float_Stack_All + 5 + pre_call_FPU_size();
   306 }
   308 // Indicate if the safepoint node needs the polling page as an input.
   309 // Since x86 does have absolute addressing, it doesn't.
   310 bool SafePointNode::needs_polling_address_input() {
   311   return false;
   312 }
   314 //
   315 // Compute padding required for nodes which need alignment
   316 //
   318 // The address of the call instruction needs to be 4-byte aligned to
   319 // ensure that it does not span a cache line so that it can be patched.
   320 int CallStaticJavaDirectNode::compute_padding(int current_offset) const {
   321   current_offset += pre_call_FPU_size();  // skip fldcw, if any
   322   current_offset += 1;      // skip call opcode byte
   323   return round_to(current_offset, alignment_required()) - current_offset;
   324 }
   326 // The address of the call instruction needs to be 4-byte aligned to
   327 // ensure that it does not span a cache line so that it can be patched.
   328 int CallStaticJavaHandleNode::compute_padding(int current_offset) const {
   329   current_offset += pre_call_FPU_size();  // skip fldcw, if any
   330   current_offset += preserve_SP_size();   // skip mov rbp, rsp
   331   current_offset += 1;      // skip call opcode byte
   332   return round_to(current_offset, alignment_required()) - current_offset;
   333 }
   335 // The address of the call instruction needs to be 4-byte aligned to
   336 // ensure that it does not span a cache line so that it can be patched.
   337 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const {
   338   current_offset += pre_call_FPU_size();  // skip fldcw, if any
   339   current_offset += 5;      // skip MOV instruction
   340   current_offset += 1;      // skip call opcode byte
   341   return round_to(current_offset, alignment_required()) - current_offset;
   342 }
   344 #ifndef PRODUCT
   345 void MachBreakpointNode::format( PhaseRegAlloc *, outputStream* st ) const {
   346   st->print("INT3");
   347 }
   348 #endif
   350 // EMIT_RM()
   351 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
   352   unsigned char c = (unsigned char)((f1 << 6) | (f2 << 3) | f3);
   353   cbuf.insts()->emit_int8(c);
   354 }
   356 // EMIT_CC()
   357 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
   358   unsigned char c = (unsigned char)( f1 | f2 );
   359   cbuf.insts()->emit_int8(c);
   360 }
   362 // EMIT_OPCODE()
   363 void emit_opcode(CodeBuffer &cbuf, int code) {
   364   cbuf.insts()->emit_int8((unsigned char) code);
   365 }
   367 // EMIT_OPCODE() w/ relocation information
   368 void emit_opcode(CodeBuffer &cbuf, int code, relocInfo::relocType reloc, int offset = 0) {
   369   cbuf.relocate(cbuf.insts_mark() + offset, reloc);
   370   emit_opcode(cbuf, code);
   371 }
   373 // EMIT_D8()
   374 void emit_d8(CodeBuffer &cbuf, int d8) {
   375   cbuf.insts()->emit_int8((unsigned char) d8);
   376 }
   378 // EMIT_D16()
   379 void emit_d16(CodeBuffer &cbuf, int d16) {
   380   cbuf.insts()->emit_int16(d16);
   381 }
   383 // EMIT_D32()
   384 void emit_d32(CodeBuffer &cbuf, int d32) {
   385   cbuf.insts()->emit_int32(d32);
   386 }
   388 // emit 32 bit value and construct relocation entry from relocInfo::relocType
   389 void emit_d32_reloc(CodeBuffer &cbuf, int d32, relocInfo::relocType reloc,
   390         int format) {
   391   cbuf.relocate(cbuf.insts_mark(), reloc, format);
   392   cbuf.insts()->emit_int32(d32);
   393 }
   395 // emit 32 bit value and construct relocation entry from RelocationHolder
   396 void emit_d32_reloc(CodeBuffer &cbuf, int d32, RelocationHolder const& rspec,
   397         int format) {
   398 #ifdef ASSERT
   399   if (rspec.reloc()->type() == relocInfo::oop_type && d32 != 0 && d32 != (int)Universe::non_oop_word()) {
   400     assert(oop(d32)->is_oop() && (ScavengeRootsInCode || !oop(d32)->is_scavengable()), "cannot embed scavengable oops in code");
   401   }
   402 #endif
   403   cbuf.relocate(cbuf.insts_mark(), rspec, format);
   404   cbuf.insts()->emit_int32(d32);
   405 }
   407 // Access stack slot for load or store
   408 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp) {
   409   emit_opcode( cbuf, opcode );               // (e.g., FILD   [ESP+src])
   410   if( -128 <= disp && disp <= 127 ) {
   411     emit_rm( cbuf, 0x01, rm_field, ESP_enc );  // R/M byte
   412     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte
   413     emit_d8 (cbuf, disp);     // Displacement  // R/M byte
   414   } else {
   415     emit_rm( cbuf, 0x02, rm_field, ESP_enc );  // R/M byte
   416     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte
   417     emit_d32(cbuf, disp);     // Displacement  // R/M byte
   418   }
   419 }
   421    // eRegI ereg, memory mem) %{    // emit_reg_mem
   422 void encode_RegMem( CodeBuffer &cbuf, int reg_encoding, int base, int index, int scale, int displace, bool displace_is_oop ) {
   423   // There is no index & no scale, use form without SIB byte
   424   if ((index == 0x4) &&
   425       (scale == 0) && (base != ESP_enc)) {
   426     // If no displacement, mode is 0x0; unless base is [EBP]
   427     if ( (displace == 0) && (base != EBP_enc) ) {
   428       emit_rm(cbuf, 0x0, reg_encoding, base);
   429     }
   430     else {                    // If 8-bit displacement, mode 0x1
   431       if ((displace >= -128) && (displace <= 127)
   432           && !(displace_is_oop) ) {
   433         emit_rm(cbuf, 0x1, reg_encoding, base);
   434         emit_d8(cbuf, displace);
   435       }
   436       else {                  // If 32-bit displacement
   437         if (base == -1) { // Special flag for absolute address
   438           emit_rm(cbuf, 0x0, reg_encoding, 0x5);
   439           // (manual lies; no SIB needed here)
   440           if ( displace_is_oop ) {
   441             emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);
   442           } else {
   443             emit_d32      (cbuf, displace);
   444           }
   445         }
   446         else {                // Normal base + offset
   447           emit_rm(cbuf, 0x2, reg_encoding, base);
   448           if ( displace_is_oop ) {
   449             emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);
   450           } else {
   451             emit_d32      (cbuf, displace);
   452           }
   453         }
   454       }
   455     }
   456   }
   457   else {                      // Else, encode with the SIB byte
   458     // If no displacement, mode is 0x0; unless base is [EBP]
   459     if (displace == 0 && (base != EBP_enc)) {  // If no displacement
   460       emit_rm(cbuf, 0x0, reg_encoding, 0x4);
   461       emit_rm(cbuf, scale, index, base);
   462     }
   463     else {                    // If 8-bit displacement, mode 0x1
   464       if ((displace >= -128) && (displace <= 127)
   465           && !(displace_is_oop) ) {
   466         emit_rm(cbuf, 0x1, reg_encoding, 0x4);
   467         emit_rm(cbuf, scale, index, base);
   468         emit_d8(cbuf, displace);
   469       }
   470       else {                  // If 32-bit displacement
   471         if (base == 0x04 ) {
   472           emit_rm(cbuf, 0x2, reg_encoding, 0x4);
   473           emit_rm(cbuf, scale, index, 0x04);
   474         } else {
   475           emit_rm(cbuf, 0x2, reg_encoding, 0x4);
   476           emit_rm(cbuf, scale, index, base);
   477         }
   478         if ( displace_is_oop ) {
   479           emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);
   480         } else {
   481           emit_d32      (cbuf, displace);
   482         }
   483       }
   484     }
   485   }
   486 }
   489 void encode_Copy( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {
   490   if( dst_encoding == src_encoding ) {
   491     // reg-reg copy, use an empty encoding
   492   } else {
   493     emit_opcode( cbuf, 0x8B );
   494     emit_rm(cbuf, 0x3, dst_encoding, src_encoding );
   495   }
   496 }
   498 void encode_CopyXD( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {
   499   if( dst_encoding == src_encoding ) {
   500     // reg-reg copy, use an empty encoding
   501   } else {
   502     MacroAssembler _masm(&cbuf);
   504     __ movdqa(as_XMMRegister(dst_encoding), as_XMMRegister(src_encoding));
   505   }
   506 }
   509 //=============================================================================
   510 const bool Matcher::constant_table_absolute_addressing = true;
   511 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
   513 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
   514   // Empty encoding
   515 }
   517 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
   518   return 0;
   519 }
   521 #ifndef PRODUCT
   522 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
   523   st->print("# MachConstantBaseNode (empty encoding)");
   524 }
   525 #endif
   528 //=============================================================================
   529 #ifndef PRODUCT
   530 void MachPrologNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
   531   Compile* C = ra_->C;
   532   if( C->in_24_bit_fp_mode() ) {
   533     st->print("FLDCW  24 bit fpu control word");
   534     st->print_cr(""); st->print("\t");
   535   }
   537   int framesize = C->frame_slots() << LogBytesPerInt;
   538   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
   539   // Remove two words for return addr and rbp,
   540   framesize -= 2*wordSize;
   542   // Calls to C2R adapters often do not accept exceptional returns.
   543   // We require that their callers must bang for them.  But be careful, because
   544   // some VM calls (such as call site linkage) can use several kilobytes of
   545   // stack.  But the stack safety zone should account for that.
   546   // See bugs 4446381, 4468289, 4497237.
   547   if (C->need_stack_bang(framesize)) {
   548     st->print_cr("# stack bang"); st->print("\t");
   549   }
   550   st->print_cr("PUSHL  EBP"); st->print("\t");
   552   if( VerifyStackAtCalls ) { // Majik cookie to verify stack depth
   553     st->print("PUSH   0xBADB100D\t# Majik cookie for stack depth check");
   554     st->print_cr(""); st->print("\t");
   555     framesize -= wordSize;
   556   }
   558   if ((C->in_24_bit_fp_mode() || VerifyStackAtCalls ) && framesize < 128 ) {
   559     if (framesize) {
   560       st->print("SUB    ESP,%d\t# Create frame",framesize);
   561     }
   562   } else {
   563     st->print("SUB    ESP,%d\t# Create frame",framesize);
   564   }
   565 }
   566 #endif
   569 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
   570   Compile* C = ra_->C;
   572   if (UseSSE >= 2 && VerifyFPU) {
   573     MacroAssembler masm(&cbuf);
   574     masm.verify_FPU(0, "FPU stack must be clean on entry");
   575   }
   577   // WARNING: Initial instruction MUST be 5 bytes or longer so that
   578   // NativeJump::patch_verified_entry will be able to patch out the entry
   579   // code safely. The fldcw is ok at 6 bytes, the push to verify stack
   580   // depth is ok at 5 bytes, the frame allocation can be either 3 or
   581   // 6 bytes. So if we don't do the fldcw or the push then we must
   582   // use the 6 byte frame allocation even if we have no frame. :-(
   583   // If method sets FPU control word do it now
   584   if( C->in_24_bit_fp_mode() ) {
   585     MacroAssembler masm(&cbuf);
   586     masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
   587   }
   589   int framesize = C->frame_slots() << LogBytesPerInt;
   590   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
   591   // Remove two words for return addr and rbp,
   592   framesize -= 2*wordSize;
   594   // Calls to C2R adapters often do not accept exceptional returns.
   595   // We require that their callers must bang for them.  But be careful, because
   596   // some VM calls (such as call site linkage) can use several kilobytes of
   597   // stack.  But the stack safety zone should account for that.
   598   // See bugs 4446381, 4468289, 4497237.
   599   if (C->need_stack_bang(framesize)) {
   600     MacroAssembler masm(&cbuf);
   601     masm.generate_stack_overflow_check(framesize);
   602   }
   604   // We always push rbp, so that on return to interpreter rbp, will be
   605   // restored correctly and we can correct the stack.
   606   emit_opcode(cbuf, 0x50 | EBP_enc);
   608   if( VerifyStackAtCalls ) { // Majik cookie to verify stack depth
   609     emit_opcode(cbuf, 0x68); // push 0xbadb100d
   610     emit_d32(cbuf, 0xbadb100d);
   611     framesize -= wordSize;
   612   }
   614   if ((C->in_24_bit_fp_mode() || VerifyStackAtCalls ) && framesize < 128 ) {
   615     if (framesize) {
   616       emit_opcode(cbuf, 0x83);   // sub  SP,#framesize
   617       emit_rm(cbuf, 0x3, 0x05, ESP_enc);
   618       emit_d8(cbuf, framesize);
   619     }
   620   } else {
   621     emit_opcode(cbuf, 0x81);   // sub  SP,#framesize
   622     emit_rm(cbuf, 0x3, 0x05, ESP_enc);
   623     emit_d32(cbuf, framesize);
   624   }
   625   C->set_frame_complete(cbuf.insts_size());
   627 #ifdef ASSERT
   628   if (VerifyStackAtCalls) {
   629     Label L;
   630     MacroAssembler masm(&cbuf);
   631     masm.push(rax);
   632     masm.mov(rax, rsp);
   633     masm.andptr(rax, StackAlignmentInBytes-1);
   634     masm.cmpptr(rax, StackAlignmentInBytes-wordSize);
   635     masm.pop(rax);
   636     masm.jcc(Assembler::equal, L);
   637     masm.stop("Stack is not properly aligned!");
   638     masm.bind(L);
   639   }
   640 #endif
   642 }
   644 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {
   645   return MachNode::size(ra_); // too many variables; just compute it the hard way
   646 }
   648 int MachPrologNode::reloc() const {
   649   return 0; // a large enough number
   650 }
   652 //=============================================================================
   653 #ifndef PRODUCT
   654 void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
   655   Compile *C = ra_->C;
   656   int framesize = C->frame_slots() << LogBytesPerInt;
   657   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
   658   // Remove two words for return addr and rbp,
   659   framesize -= 2*wordSize;
   661   if( C->in_24_bit_fp_mode() ) {
   662     st->print("FLDCW  standard control word");
   663     st->cr(); st->print("\t");
   664   }
   665   if( framesize ) {
   666     st->print("ADD    ESP,%d\t# Destroy frame",framesize);
   667     st->cr(); st->print("\t");
   668   }
   669   st->print_cr("POPL   EBP"); st->print("\t");
   670   if( do_polling() && C->is_method_compilation() ) {
   671     st->print("TEST   PollPage,EAX\t! Poll Safepoint");
   672     st->cr(); st->print("\t");
   673   }
   674 }
   675 #endif
   677 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
   678   Compile *C = ra_->C;
   680   // If method set FPU control word, restore to standard control word
   681   if( C->in_24_bit_fp_mode() ) {
   682     MacroAssembler masm(&cbuf);
   683     masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
   684   }
   686   int framesize = C->frame_slots() << LogBytesPerInt;
   687   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
   688   // Remove two words for return addr and rbp,
   689   framesize -= 2*wordSize;
   691   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
   693   if( framesize >= 128 ) {
   694     emit_opcode(cbuf, 0x81); // add  SP, #framesize
   695     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
   696     emit_d32(cbuf, framesize);
   697   }
   698   else if( framesize ) {
   699     emit_opcode(cbuf, 0x83); // add  SP, #framesize
   700     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
   701     emit_d8(cbuf, framesize);
   702   }
   704   emit_opcode(cbuf, 0x58 | EBP_enc);
   706   if( do_polling() && C->is_method_compilation() ) {
   707     cbuf.relocate(cbuf.insts_end(), relocInfo::poll_return_type, 0);
   708     emit_opcode(cbuf,0x85);
   709     emit_rm(cbuf, 0x0, EAX_enc, 0x5); // EAX
   710     emit_d32(cbuf, (intptr_t)os::get_polling_page());
   711   }
   712 }
   714 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
   715   Compile *C = ra_->C;
   716   // If method set FPU control word, restore to standard control word
   717   int size = C->in_24_bit_fp_mode() ? 6 : 0;
   718   if( do_polling() && C->is_method_compilation() ) size += 6;
   720   int framesize = C->frame_slots() << LogBytesPerInt;
   721   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
   722   // Remove two words for return addr and rbp,
   723   framesize -= 2*wordSize;
   725   size++; // popl rbp,
   727   if( framesize >= 128 ) {
   728     size += 6;
   729   } else {
   730     size += framesize ? 3 : 0;
   731   }
   732   return size;
   733 }
   735 int MachEpilogNode::reloc() const {
   736   return 0; // a large enough number
   737 }
   739 const Pipeline * MachEpilogNode::pipeline() const {
   740   return MachNode::pipeline_class();
   741 }
   743 int MachEpilogNode::safepoint_offset() const { return 0; }
   745 //=============================================================================
   747 enum RC { rc_bad, rc_int, rc_float, rc_xmm, rc_stack };
   748 static enum RC rc_class( OptoReg::Name reg ) {
   750   if( !OptoReg::is_valid(reg)  ) return rc_bad;
   751   if (OptoReg::is_stack(reg)) return rc_stack;
   753   VMReg r = OptoReg::as_VMReg(reg);
   754   if (r->is_Register()) return rc_int;
   755   if (r->is_FloatRegister()) {
   756     assert(UseSSE < 2, "shouldn't be used in SSE2+ mode");
   757     return rc_float;
   758   }
   759   assert(r->is_XMMRegister(), "must be");
   760   return rc_xmm;
   761 }
   763 static int impl_helper( CodeBuffer *cbuf, bool do_size, bool is_load, int offset, int reg,
   764                         int opcode, const char *op_str, int size, outputStream* st ) {
   765   if( cbuf ) {
   766     emit_opcode  (*cbuf, opcode );
   767     encode_RegMem(*cbuf, Matcher::_regEncode[reg], ESP_enc, 0x4, 0, offset, false);
   768 #ifndef PRODUCT
   769   } else if( !do_size ) {
   770     if( size != 0 ) st->print("\n\t");
   771     if( opcode == 0x8B || opcode == 0x89 ) { // MOV
   772       if( is_load ) st->print("%s   %s,[ESP + #%d]",op_str,Matcher::regName[reg],offset);
   773       else          st->print("%s   [ESP + #%d],%s",op_str,offset,Matcher::regName[reg]);
   774     } else { // FLD, FST, PUSH, POP
   775       st->print("%s [ESP + #%d]",op_str,offset);
   776     }
   777 #endif
   778   }
   779   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
   780   return size+3+offset_size;
   781 }
   783 // Helper for XMM registers.  Extra opcode bits, limited syntax.
   784 static int impl_x_helper( CodeBuffer *cbuf, bool do_size, bool is_load,
   785                          int offset, int reg_lo, int reg_hi, int size, outputStream* st ) {
   786   if( cbuf ) {
   787     if( reg_lo+1 == reg_hi ) { // double move?
   788       if( is_load && !UseXmmLoadAndClearUpper )
   789         emit_opcode(*cbuf, 0x66 ); // use 'movlpd' for load
   790       else
   791         emit_opcode(*cbuf, 0xF2 ); // use 'movsd' otherwise
   792     } else {
   793       emit_opcode(*cbuf, 0xF3 );
   794     }
   795     emit_opcode(*cbuf, 0x0F );
   796     if( reg_lo+1 == reg_hi && is_load && !UseXmmLoadAndClearUpper )
   797       emit_opcode(*cbuf, 0x12 );   // use 'movlpd' for load
   798     else
   799       emit_opcode(*cbuf, is_load ? 0x10 : 0x11 );
   800     encode_RegMem(*cbuf, Matcher::_regEncode[reg_lo], ESP_enc, 0x4, 0, offset, false);
   801 #ifndef PRODUCT
   802   } else if( !do_size ) {
   803     if( size != 0 ) st->print("\n\t");
   804     if( reg_lo+1 == reg_hi ) { // double move?
   805       if( is_load ) st->print("%s %s,[ESP + #%d]",
   806                                UseXmmLoadAndClearUpper ? "MOVSD " : "MOVLPD",
   807                                Matcher::regName[reg_lo], offset);
   808       else          st->print("MOVSD  [ESP + #%d],%s",
   809                                offset, Matcher::regName[reg_lo]);
   810     } else {
   811       if( is_load ) st->print("MOVSS  %s,[ESP + #%d]",
   812                                Matcher::regName[reg_lo], offset);
   813       else          st->print("MOVSS  [ESP + #%d],%s",
   814                                offset, Matcher::regName[reg_lo]);
   815     }
   816 #endif
   817   }
   818   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
   819   return size+5+offset_size;
   820 }
   823 static int impl_movx_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
   824                             int src_hi, int dst_hi, int size, outputStream* st ) {
   825   if( UseXmmRegToRegMoveAll ) {//Use movaps,movapd to move between xmm registers
   826     if( cbuf ) {
   827       if( (src_lo+1 == src_hi && dst_lo+1 == dst_hi) ) {
   828         emit_opcode(*cbuf, 0x66 );
   829       }
   830       emit_opcode(*cbuf, 0x0F );
   831       emit_opcode(*cbuf, 0x28 );
   832       emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst_lo], Matcher::_regEncode[src_lo] );
   833 #ifndef PRODUCT
   834     } else if( !do_size ) {
   835       if( size != 0 ) st->print("\n\t");
   836       if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?
   837         st->print("MOVAPD %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
   838       } else {
   839         st->print("MOVAPS %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
   840       }
   841 #endif
   842     }
   843     return size + ((src_lo+1 == src_hi && dst_lo+1 == dst_hi) ? 4 : 3);
   844   } else {
   845     if( cbuf ) {
   846       emit_opcode(*cbuf, (src_lo+1 == src_hi && dst_lo+1 == dst_hi) ? 0xF2 : 0xF3 );
   847       emit_opcode(*cbuf, 0x0F );
   848       emit_opcode(*cbuf, 0x10 );
   849       emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst_lo], Matcher::_regEncode[src_lo] );
   850 #ifndef PRODUCT
   851     } else if( !do_size ) {
   852       if( size != 0 ) st->print("\n\t");
   853       if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?
   854         st->print("MOVSD  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
   855       } else {
   856         st->print("MOVSS  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
   857       }
   858 #endif
   859     }
   860     return size+4;
   861   }
   862 }
   864 static int impl_movgpr2x_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
   865                             int src_hi, int dst_hi, int size, outputStream* st ) {
   866   // 32-bit
   867   if (cbuf) {
   868     emit_opcode(*cbuf, 0x66);
   869     emit_opcode(*cbuf, 0x0F);
   870     emit_opcode(*cbuf, 0x6E);
   871     emit_rm(*cbuf, 0x3, Matcher::_regEncode[dst_lo] & 7, Matcher::_regEncode[src_lo] & 7);
   872 #ifndef PRODUCT
   873   } else if (!do_size) {
   874     st->print("movdl   %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);
   875 #endif
   876   }
   877   return 4;
   878 }
   881 static int impl_movx2gpr_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
   882                                  int src_hi, int dst_hi, int size, outputStream* st ) {
   883   // 32-bit
   884   if (cbuf) {
   885     emit_opcode(*cbuf, 0x66);
   886     emit_opcode(*cbuf, 0x0F);
   887     emit_opcode(*cbuf, 0x7E);
   888     emit_rm(*cbuf, 0x3, Matcher::_regEncode[src_lo] & 7, Matcher::_regEncode[dst_lo] & 7);
   889 #ifndef PRODUCT
   890   } else if (!do_size) {
   891     st->print("movdl   %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);
   892 #endif
   893   }
   894   return 4;
   895 }
   897 static int impl_mov_helper( CodeBuffer *cbuf, bool do_size, int src, int dst, int size, outputStream* st ) {
   898   if( cbuf ) {
   899     emit_opcode(*cbuf, 0x8B );
   900     emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst], Matcher::_regEncode[src] );
   901 #ifndef PRODUCT
   902   } else if( !do_size ) {
   903     if( size != 0 ) st->print("\n\t");
   904     st->print("MOV    %s,%s",Matcher::regName[dst],Matcher::regName[src]);
   905 #endif
   906   }
   907   return size+2;
   908 }
   910 static int impl_fp_store_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int src_hi, int dst_lo, int dst_hi,
   911                                  int offset, int size, outputStream* st ) {
   912   if( src_lo != FPR1L_num ) {      // Move value to top of FP stack, if not already there
   913     if( cbuf ) {
   914       emit_opcode( *cbuf, 0xD9 );  // FLD (i.e., push it)
   915       emit_d8( *cbuf, 0xC0-1+Matcher::_regEncode[src_lo] );
   916 #ifndef PRODUCT
   917     } else if( !do_size ) {
   918       if( size != 0 ) st->print("\n\t");
   919       st->print("FLD    %s",Matcher::regName[src_lo]);
   920 #endif
   921     }
   922     size += 2;
   923   }
   925   int st_op = (src_lo != FPR1L_num) ? EBX_num /*store & pop*/ : EDX_num /*store no pop*/;
   926   const char *op_str;
   927   int op;
   928   if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double store?
   929     op_str = (src_lo != FPR1L_num) ? "FSTP_D" : "FST_D ";
   930     op = 0xDD;
   931   } else {                   // 32-bit store
   932     op_str = (src_lo != FPR1L_num) ? "FSTP_S" : "FST_S ";
   933     op = 0xD9;
   934     assert( !OptoReg::is_valid(src_hi) && !OptoReg::is_valid(dst_hi), "no non-adjacent float-stores" );
   935   }
   937   return impl_helper(cbuf,do_size,false,offset,st_op,op,op_str,size, st);
   938 }
   940 uint MachSpillCopyNode::implementation( CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream* st ) const {
   941   // Get registers to move
   942   OptoReg::Name src_second = ra_->get_reg_second(in(1));
   943   OptoReg::Name src_first = ra_->get_reg_first(in(1));
   944   OptoReg::Name dst_second = ra_->get_reg_second(this );
   945   OptoReg::Name dst_first = ra_->get_reg_first(this );
   947   enum RC src_second_rc = rc_class(src_second);
   948   enum RC src_first_rc = rc_class(src_first);
   949   enum RC dst_second_rc = rc_class(dst_second);
   950   enum RC dst_first_rc = rc_class(dst_first);
   952   assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );
   954   // Generate spill code!
   955   int size = 0;
   957   if( src_first == dst_first && src_second == dst_second )
   958     return size;            // Self copy, no move
   960   // --------------------------------------
   961   // Check for mem-mem move.  push/pop to move.
   962   if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
   963     if( src_second == dst_first ) { // overlapping stack copy ranges
   964       assert( src_second_rc == rc_stack && dst_second_rc == rc_stack, "we only expect a stk-stk copy here" );
   965       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);
   966       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);
   967       src_second_rc = dst_second_rc = rc_bad;  // flag as already moved the second bits
   968     }
   969     // move low bits
   970     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),ESI_num,0xFF,"PUSH  ",size, st);
   971     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),EAX_num,0x8F,"POP   ",size, st);
   972     if( src_second_rc == rc_stack && dst_second_rc == rc_stack ) { // mov second bits
   973       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);
   974       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);
   975     }
   976     return size;
   977   }
   979   // --------------------------------------
   980   // Check for integer reg-reg copy
   981   if( src_first_rc == rc_int && dst_first_rc == rc_int )
   982     size = impl_mov_helper(cbuf,do_size,src_first,dst_first,size, st);
   984   // Check for integer store
   985   if( src_first_rc == rc_int && dst_first_rc == rc_stack )
   986     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first,0x89,"MOV ",size, st);
   988   // Check for integer load
   989   if( dst_first_rc == rc_int && src_first_rc == rc_stack )
   990     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first,0x8B,"MOV ",size, st);
   992   // Check for integer reg-xmm reg copy
   993   if( src_first_rc == rc_int && dst_first_rc == rc_xmm ) {
   994     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),
   995             "no 64 bit integer-float reg moves" );
   996     return impl_movgpr2x_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);
   997   }
   998   // --------------------------------------
   999   // Check for float reg-reg copy
  1000   if( src_first_rc == rc_float && dst_first_rc == rc_float ) {
  1001     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||
  1002             (src_first+1 == src_second && dst_first+1 == dst_second), "no non-adjacent float-moves" );
  1003     if( cbuf ) {
  1005       // Note the mucking with the register encode to compensate for the 0/1
  1006       // indexing issue mentioned in a comment in the reg_def sections
  1007       // for FPR registers many lines above here.
  1009       if( src_first != FPR1L_num ) {
  1010         emit_opcode  (*cbuf, 0xD9 );           // FLD    ST(i)
  1011         emit_d8      (*cbuf, 0xC0+Matcher::_regEncode[src_first]-1 );
  1012         emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)
  1013         emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );
  1014      } else {
  1015         emit_opcode  (*cbuf, 0xDD );           // FST    ST(i)
  1016         emit_d8      (*cbuf, 0xD0+Matcher::_regEncode[dst_first]-1 );
  1018 #ifndef PRODUCT
  1019     } else if( !do_size ) {
  1020       if( size != 0 ) st->print("\n\t");
  1021       if( src_first != FPR1L_num ) st->print("FLD    %s\n\tFSTP   %s",Matcher::regName[src_first],Matcher::regName[dst_first]);
  1022       else                      st->print(             "FST    %s",                            Matcher::regName[dst_first]);
  1023 #endif
  1025     return size + ((src_first != FPR1L_num) ? 2+2 : 2);
  1028   // Check for float store
  1029   if( src_first_rc == rc_float && dst_first_rc == rc_stack ) {
  1030     return impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,ra_->reg2offset(dst_first),size, st);
  1033   // Check for float load
  1034   if( dst_first_rc == rc_float && src_first_rc == rc_stack ) {
  1035     int offset = ra_->reg2offset(src_first);
  1036     const char *op_str;
  1037     int op;
  1038     if( src_first+1 == src_second && dst_first+1 == dst_second ) { // double load?
  1039       op_str = "FLD_D";
  1040       op = 0xDD;
  1041     } else {                   // 32-bit load
  1042       op_str = "FLD_S";
  1043       op = 0xD9;
  1044       assert( src_second_rc == rc_bad && dst_second_rc == rc_bad, "no non-adjacent float-loads" );
  1046     if( cbuf ) {
  1047       emit_opcode  (*cbuf, op );
  1048       encode_RegMem(*cbuf, 0x0, ESP_enc, 0x4, 0, offset, false);
  1049       emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)
  1050       emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );
  1051 #ifndef PRODUCT
  1052     } else if( !do_size ) {
  1053       if( size != 0 ) st->print("\n\t");
  1054       st->print("%s  ST,[ESP + #%d]\n\tFSTP   %s",op_str, offset,Matcher::regName[dst_first]);
  1055 #endif
  1057     int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
  1058     return size + 3+offset_size+2;
  1061   // Check for xmm reg-reg copy
  1062   if( src_first_rc == rc_xmm && dst_first_rc == rc_xmm ) {
  1063     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||
  1064             (src_first+1 == src_second && dst_first+1 == dst_second),
  1065             "no non-adjacent float-moves" );
  1066     return impl_movx_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);
  1069   // Check for xmm reg-integer reg copy
  1070   if( src_first_rc == rc_xmm && dst_first_rc == rc_int ) {
  1071     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),
  1072             "no 64 bit float-integer reg moves" );
  1073     return impl_movx2gpr_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);
  1076   // Check for xmm store
  1077   if( src_first_rc == rc_xmm && dst_first_rc == rc_stack ) {
  1078     return impl_x_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first, src_second, size, st);
  1081   // Check for float xmm load
  1082   if( dst_first_rc == rc_xmm && src_first_rc == rc_stack ) {
  1083     return impl_x_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first, dst_second, size, st);
  1086   // Copy from float reg to xmm reg
  1087   if( dst_first_rc == rc_xmm && src_first_rc == rc_float ) {
  1088     // copy to the top of stack from floating point reg
  1089     // and use LEA to preserve flags
  1090     if( cbuf ) {
  1091       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP-8]
  1092       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);
  1093       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);
  1094       emit_d8(*cbuf,0xF8);
  1095 #ifndef PRODUCT
  1096     } else if( !do_size ) {
  1097       if( size != 0 ) st->print("\n\t");
  1098       st->print("LEA    ESP,[ESP-8]");
  1099 #endif
  1101     size += 4;
  1103     size = impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,0,size, st);
  1105     // Copy from the temp memory to the xmm reg.
  1106     size = impl_x_helper(cbuf,do_size,true ,0,dst_first, dst_second, size, st);
  1108     if( cbuf ) {
  1109       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP+8]
  1110       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);
  1111       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);
  1112       emit_d8(*cbuf,0x08);
  1113 #ifndef PRODUCT
  1114     } else if( !do_size ) {
  1115       if( size != 0 ) st->print("\n\t");
  1116       st->print("LEA    ESP,[ESP+8]");
  1117 #endif
  1119     size += 4;
  1120     return size;
  1123   assert( size > 0, "missed a case" );
  1125   // --------------------------------------------------------------------
  1126   // Check for second bits still needing moving.
  1127   if( src_second == dst_second )
  1128     return size;               // Self copy; no move
  1129   assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );
  1131   // Check for second word int-int move
  1132   if( src_second_rc == rc_int && dst_second_rc == rc_int )
  1133     return impl_mov_helper(cbuf,do_size,src_second,dst_second,size, st);
  1135   // Check for second word integer store
  1136   if( src_second_rc == rc_int && dst_second_rc == rc_stack )
  1137     return impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),src_second,0x89,"MOV ",size, st);
  1139   // Check for second word integer load
  1140   if( dst_second_rc == rc_int && src_second_rc == rc_stack )
  1141     return impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),dst_second,0x8B,"MOV ",size, st);
  1144   Unimplemented();
  1147 #ifndef PRODUCT
  1148 void MachSpillCopyNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
  1149   implementation( NULL, ra_, false, st );
  1151 #endif
  1153 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  1154   implementation( &cbuf, ra_, false, NULL );
  1157 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
  1158   return implementation( NULL, ra_, true, NULL );
  1161 //=============================================================================
  1162 #ifndef PRODUCT
  1163 void MachNopNode::format( PhaseRegAlloc *, outputStream* st ) const {
  1164   st->print("NOP \t# %d bytes pad for loops and calls", _count);
  1166 #endif
  1168 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc * ) const {
  1169   MacroAssembler _masm(&cbuf);
  1170   __ nop(_count);
  1173 uint MachNopNode::size(PhaseRegAlloc *) const {
  1174   return _count;
  1178 //=============================================================================
  1179 #ifndef PRODUCT
  1180 void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
  1181   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
  1182   int reg = ra_->get_reg_first(this);
  1183   st->print("LEA    %s,[ESP + #%d]",Matcher::regName[reg],offset);
  1185 #endif
  1187 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  1188   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
  1189   int reg = ra_->get_encode(this);
  1190   if( offset >= 128 ) {
  1191     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]
  1192     emit_rm(cbuf, 0x2, reg, 0x04);
  1193     emit_rm(cbuf, 0x0, 0x04, ESP_enc);
  1194     emit_d32(cbuf, offset);
  1196   else {
  1197     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]
  1198     emit_rm(cbuf, 0x1, reg, 0x04);
  1199     emit_rm(cbuf, 0x0, 0x04, ESP_enc);
  1200     emit_d8(cbuf, offset);
  1204 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
  1205   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
  1206   if( offset >= 128 ) {
  1207     return 7;
  1209   else {
  1210     return 4;
  1214 //=============================================================================
  1216 // emit call stub, compiled java to interpreter
  1217 void emit_java_to_interp(CodeBuffer &cbuf ) {
  1218   // Stub is fixed up when the corresponding call is converted from calling
  1219   // compiled code to calling interpreted code.
  1220   // mov rbx,0
  1221   // jmp -1
  1223   address mark = cbuf.insts_mark();  // get mark within main instrs section
  1225   // Note that the code buffer's insts_mark is always relative to insts.
  1226   // That's why we must use the macroassembler to generate a stub.
  1227   MacroAssembler _masm(&cbuf);
  1229   address base =
  1230   __ start_a_stub(Compile::MAX_stubs_size);
  1231   if (base == NULL)  return;  // CodeBuffer::expand failed
  1232   // static stub relocation stores the instruction address of the call
  1233   __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM32);
  1234   // static stub relocation also tags the methodOop in the code-stream.
  1235   __ movoop(rbx, (jobject)NULL);  // method is zapped till fixup time
  1236   // This is recognized as unresolved by relocs/nativeInst/ic code
  1237   __ jump(RuntimeAddress(__ pc()));
  1239   __ end_a_stub();
  1240   // Update current stubs pointer and restore insts_end.
  1242 // size of call stub, compiled java to interpretor
  1243 uint size_java_to_interp() {
  1244   return 10;  // movl; jmp
  1246 // relocation entries for call stub, compiled java to interpretor
  1247 uint reloc_java_to_interp() {
  1248   return 4;  // 3 in emit_java_to_interp + 1 in Java_Static_Call
  1251 //=============================================================================
  1252 #ifndef PRODUCT
  1253 void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
  1254   st->print_cr(  "CMP    EAX,[ECX+4]\t# Inline cache check");
  1255   st->print_cr("\tJNE    SharedRuntime::handle_ic_miss_stub");
  1256   st->print_cr("\tNOP");
  1257   st->print_cr("\tNOP");
  1258   if( !OptoBreakpoint )
  1259     st->print_cr("\tNOP");
  1261 #endif
  1263 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  1264   MacroAssembler masm(&cbuf);
  1265 #ifdef ASSERT
  1266   uint insts_size = cbuf.insts_size();
  1267 #endif
  1268   masm.cmpptr(rax, Address(rcx, oopDesc::klass_offset_in_bytes()));
  1269   masm.jump_cc(Assembler::notEqual,
  1270                RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
  1271   /* WARNING these NOPs are critical so that verified entry point is properly
  1272      aligned for patching by NativeJump::patch_verified_entry() */
  1273   int nops_cnt = 2;
  1274   if( !OptoBreakpoint ) // Leave space for int3
  1275      nops_cnt += 1;
  1276   masm.nop(nops_cnt);
  1278   assert(cbuf.insts_size() - insts_size == size(ra_), "checking code size of inline cache node");
  1281 uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
  1282   return OptoBreakpoint ? 11 : 12;
  1286 //=============================================================================
  1287 uint size_exception_handler() {
  1288   // NativeCall instruction size is the same as NativeJump.
  1289   // exception handler starts out as jump and can be patched to
  1290   // a call be deoptimization.  (4932387)
  1291   // Note that this value is also credited (in output.cpp) to
  1292   // the size of the code section.
  1293   return NativeJump::instruction_size;
  1296 // Emit exception handler code.  Stuff framesize into a register
  1297 // and call a VM stub routine.
  1298 int emit_exception_handler(CodeBuffer& cbuf) {
  1300   // Note that the code buffer's insts_mark is always relative to insts.
  1301   // That's why we must use the macroassembler to generate a handler.
  1302   MacroAssembler _masm(&cbuf);
  1303   address base =
  1304   __ start_a_stub(size_exception_handler());
  1305   if (base == NULL)  return 0;  // CodeBuffer::expand failed
  1306   int offset = __ offset();
  1307   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
  1308   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
  1309   __ end_a_stub();
  1310   return offset;
  1313 uint size_deopt_handler() {
  1314   // NativeCall instruction size is the same as NativeJump.
  1315   // exception handler starts out as jump and can be patched to
  1316   // a call be deoptimization.  (4932387)
  1317   // Note that this value is also credited (in output.cpp) to
  1318   // the size of the code section.
  1319   return 5 + NativeJump::instruction_size; // pushl(); jmp;
  1322 // Emit deopt handler code.
  1323 int emit_deopt_handler(CodeBuffer& cbuf) {
  1325   // Note that the code buffer's insts_mark is always relative to insts.
  1326   // That's why we must use the macroassembler to generate a handler.
  1327   MacroAssembler _masm(&cbuf);
  1328   address base =
  1329   __ start_a_stub(size_exception_handler());
  1330   if (base == NULL)  return 0;  // CodeBuffer::expand failed
  1331   int offset = __ offset();
  1332   InternalAddress here(__ pc());
  1333   __ pushptr(here.addr());
  1335   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
  1336   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
  1337   __ end_a_stub();
  1338   return offset;
  1342 const bool Matcher::match_rule_supported(int opcode) {
  1343   if (!has_match_rule(opcode))
  1344     return false;
  1346   return true;  // Per default match rules are supported.
  1349 int Matcher::regnum_to_fpu_offset(int regnum) {
  1350   return regnum - 32; // The FP registers are in the second chunk
  1353 // This is UltraSparc specific, true just means we have fast l2f conversion
  1354 const bool Matcher::convL2FSupported(void) {
  1355   return true;
  1358 // Vector width in bytes
  1359 const uint Matcher::vector_width_in_bytes(void) {
  1360   return UseSSE >= 2 ? 8 : 0;
  1363 // Vector ideal reg
  1364 const uint Matcher::vector_ideal_reg(void) {
  1365   return Op_RegD;
  1368 // Is this branch offset short enough that a short branch can be used?
  1369 //
  1370 // NOTE: If the platform does not provide any short branch variants, then
  1371 //       this method should return false for offset 0.
  1372 bool Matcher::is_short_branch_offset(int rule, int offset) {
  1373   // the short version of jmpConUCF2 contains multiple branches,
  1374   // making the reach slightly less
  1375   if (rule == jmpConUCF2_rule)
  1376     return (-126 <= offset && offset <= 125);
  1377   return (-128 <= offset && offset <= 127);
  1380 const bool Matcher::isSimpleConstant64(jlong value) {
  1381   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
  1382   return false;
  1385 // The ecx parameter to rep stos for the ClearArray node is in dwords.
  1386 const bool Matcher::init_array_count_is_in_bytes = false;
  1388 // Threshold size for cleararray.
  1389 const int Matcher::init_array_short_size = 8 * BytesPerLong;
  1391 // Should the Matcher clone shifts on addressing modes, expecting them to
  1392 // be subsumed into complex addressing expressions or compute them into
  1393 // registers?  True for Intel but false for most RISCs
  1394 const bool Matcher::clone_shift_expressions = true;
  1396 // Do we need to mask the count passed to shift instructions or does
  1397 // the cpu only look at the lower 5/6 bits anyway?
  1398 const bool Matcher::need_masked_shift_count = false;
  1400 bool Matcher::narrow_oop_use_complex_address() {
  1401   ShouldNotCallThis();
  1402   return true;
  1406 // Is it better to copy float constants, or load them directly from memory?
  1407 // Intel can load a float constant from a direct address, requiring no
  1408 // extra registers.  Most RISCs will have to materialize an address into a
  1409 // register first, so they would do better to copy the constant from stack.
  1410 const bool Matcher::rematerialize_float_constants = true;
  1412 // If CPU can load and store mis-aligned doubles directly then no fixup is
  1413 // needed.  Else we split the double into 2 integer pieces and move it
  1414 // piece-by-piece.  Only happens when passing doubles into C code as the
  1415 // Java calling convention forces doubles to be aligned.
  1416 const bool Matcher::misaligned_doubles_ok = true;
  1419 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
  1420   // Get the memory operand from the node
  1421   uint numopnds = node->num_opnds();        // Virtual call for number of operands
  1422   uint skipped  = node->oper_input_base();  // Sum of leaves skipped so far
  1423   assert( idx >= skipped, "idx too low in pd_implicit_null_fixup" );
  1424   uint opcnt     = 1;                 // First operand
  1425   uint num_edges = node->_opnds[1]->num_edges(); // leaves for first operand
  1426   while( idx >= skipped+num_edges ) {
  1427     skipped += num_edges;
  1428     opcnt++;                          // Bump operand count
  1429     assert( opcnt < numopnds, "Accessing non-existent operand" );
  1430     num_edges = node->_opnds[opcnt]->num_edges(); // leaves for next operand
  1433   MachOper *memory = node->_opnds[opcnt];
  1434   MachOper *new_memory = NULL;
  1435   switch (memory->opcode()) {
  1436   case DIRECT:
  1437   case INDOFFSET32X:
  1438     // No transformation necessary.
  1439     return;
  1440   case INDIRECT:
  1441     new_memory = new (C) indirect_win95_safeOper( );
  1442     break;
  1443   case INDOFFSET8:
  1444     new_memory = new (C) indOffset8_win95_safeOper(memory->disp(NULL, NULL, 0));
  1445     break;
  1446   case INDOFFSET32:
  1447     new_memory = new (C) indOffset32_win95_safeOper(memory->disp(NULL, NULL, 0));
  1448     break;
  1449   case INDINDEXOFFSET:
  1450     new_memory = new (C) indIndexOffset_win95_safeOper(memory->disp(NULL, NULL, 0));
  1451     break;
  1452   case INDINDEXSCALE:
  1453     new_memory = new (C) indIndexScale_win95_safeOper(memory->scale());
  1454     break;
  1455   case INDINDEXSCALEOFFSET:
  1456     new_memory = new (C) indIndexScaleOffset_win95_safeOper(memory->scale(), memory->disp(NULL, NULL, 0));
  1457     break;
  1458   case LOAD_LONG_INDIRECT:
  1459   case LOAD_LONG_INDOFFSET32:
  1460     // Does not use EBP as address register, use { EDX, EBX, EDI, ESI}
  1461     return;
  1462   default:
  1463     assert(false, "unexpected memory operand in pd_implicit_null_fixup()");
  1464     return;
  1466   node->_opnds[opcnt] = new_memory;
  1469 // Advertise here if the CPU requires explicit rounding operations
  1470 // to implement the UseStrictFP mode.
  1471 const bool Matcher::strict_fp_requires_explicit_rounding = true;
  1473 // Are floats conerted to double when stored to stack during deoptimization?
  1474 // On x32 it is stored with convertion only when FPU is used for floats.
  1475 bool Matcher::float_in_double() { return (UseSSE == 0); }
  1477 // Do ints take an entire long register or just half?
  1478 const bool Matcher::int_in_long = false;
  1480 // Return whether or not this register is ever used as an argument.  This
  1481 // function is used on startup to build the trampoline stubs in generateOptoStub.
  1482 // Registers not mentioned will be killed by the VM call in the trampoline, and
  1483 // arguments in those registers not be available to the callee.
  1484 bool Matcher::can_be_java_arg( int reg ) {
  1485   if(  reg == ECX_num   || reg == EDX_num   ) return true;
  1486   if( (reg == XMM0a_num || reg == XMM1a_num) && UseSSE>=1 ) return true;
  1487   if( (reg == XMM0b_num || reg == XMM1b_num) && UseSSE>=2 ) return true;
  1488   return false;
  1491 bool Matcher::is_spillable_arg( int reg ) {
  1492   return can_be_java_arg(reg);
  1495 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
  1496   // Use hardware integer DIV instruction when
  1497   // it is faster than a code which use multiply.
  1498   // Only when constant divisor fits into 32 bit
  1499   // (min_jint is excluded to get only correct
  1500   // positive 32 bit values from negative).
  1501   return VM_Version::has_fast_idiv() &&
  1502          (divisor == (int)divisor && divisor != min_jint);
  1505 // Register for DIVI projection of divmodI
  1506 RegMask Matcher::divI_proj_mask() {
  1507   return EAX_REG_mask;
  1510 // Register for MODI projection of divmodI
  1511 RegMask Matcher::modI_proj_mask() {
  1512   return EDX_REG_mask;
  1515 // Register for DIVL projection of divmodL
  1516 RegMask Matcher::divL_proj_mask() {
  1517   ShouldNotReachHere();
  1518   return RegMask();
  1521 // Register for MODL projection of divmodL
  1522 RegMask Matcher::modL_proj_mask() {
  1523   ShouldNotReachHere();
  1524   return RegMask();
  1527 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
  1528   return EBP_REG_mask;
  1531 // Returns true if the high 32 bits of the value is known to be zero.
  1532 bool is_operand_hi32_zero(Node* n) {
  1533   int opc = n->Opcode();
  1534   if (opc == Op_LoadUI2L) {
  1535     return true;
  1537   if (opc == Op_AndL) {
  1538     Node* o2 = n->in(2);
  1539     if (o2->is_Con() && (o2->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
  1540       return true;
  1543   if (opc == Op_ConL && (n->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
  1544     return true;
  1546   return false;
  1549 %}
  1551 //----------ENCODING BLOCK-----------------------------------------------------
  1552 // This block specifies the encoding classes used by the compiler to output
  1553 // byte streams.  Encoding classes generate functions which are called by
  1554 // Machine Instruction Nodes in order to generate the bit encoding of the
  1555 // instruction.  Operands specify their base encoding interface with the
  1556 // interface keyword.  There are currently supported four interfaces,
  1557 // REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER.  REG_INTER causes an
  1558 // operand to generate a function which returns its register number when
  1559 // queried.   CONST_INTER causes an operand to generate a function which
  1560 // returns the value of the constant when queried.  MEMORY_INTER causes an
  1561 // operand to generate four functions which return the Base Register, the
  1562 // Index Register, the Scale Value, and the Offset Value of the operand when
  1563 // queried.  COND_INTER causes an operand to generate six functions which
  1564 // return the encoding code (ie - encoding bits for the instruction)
  1565 // associated with each basic boolean condition for a conditional instruction.
  1566 // Instructions specify two basic values for encoding.  They use the
  1567 // ins_encode keyword to specify their encoding class (which must be one of
  1568 // the class names specified in the encoding block), and they use the
  1569 // opcode keyword to specify, in order, their primary, secondary, and
  1570 // tertiary opcode.  Only the opcode sections which a particular instruction
  1571 // needs for encoding need to be specified.
  1572 encode %{
  1573   // Build emit functions for each basic byte or larger field in the intel
  1574   // encoding scheme (opcode, rm, sib, immediate), and call them from C++
  1575   // code in the enc_class source block.  Emit functions will live in the
  1576   // main source block for now.  In future, we can generalize this by
  1577   // adding a syntax that specifies the sizes of fields in an order,
  1578   // so that the adlc can build the emit functions automagically
  1580   // Emit primary opcode
  1581   enc_class OpcP %{
  1582     emit_opcode(cbuf, $primary);
  1583   %}
  1585   // Emit secondary opcode
  1586   enc_class OpcS %{
  1587     emit_opcode(cbuf, $secondary);
  1588   %}
  1590   // Emit opcode directly
  1591   enc_class Opcode(immI d8) %{
  1592     emit_opcode(cbuf, $d8$$constant);
  1593   %}
  1595   enc_class SizePrefix %{
  1596     emit_opcode(cbuf,0x66);
  1597   %}
  1599   enc_class RegReg (eRegI dst, eRegI src) %{    // RegReg(Many)
  1600     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  1601   %}
  1603   enc_class OpcRegReg (immI opcode, eRegI dst, eRegI src) %{    // OpcRegReg(Many)
  1604     emit_opcode(cbuf,$opcode$$constant);
  1605     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  1606   %}
  1608   enc_class mov_r32_imm0( eRegI dst ) %{
  1609     emit_opcode( cbuf, 0xB8 + $dst$$reg ); // 0xB8+ rd   -- MOV r32  ,imm32
  1610     emit_d32   ( cbuf, 0x0  );             //                         imm32==0x0
  1611   %}
  1613   enc_class cdq_enc %{
  1614     // Full implementation of Java idiv and irem; checks for
  1615     // special case as described in JVM spec., p.243 & p.271.
  1616     //
  1617     //         normal case                           special case
  1618     //
  1619     // input : rax,: dividend                         min_int
  1620     //         reg: divisor                          -1
  1621     //
  1622     // output: rax,: quotient  (= rax, idiv reg)       min_int
  1623     //         rdx: remainder (= rax, irem reg)       0
  1624     //
  1625     //  Code sequnce:
  1626     //
  1627     //  81 F8 00 00 00 80    cmp         rax,80000000h
  1628     //  0F 85 0B 00 00 00    jne         normal_case
  1629     //  33 D2                xor         rdx,edx
  1630     //  83 F9 FF             cmp         rcx,0FFh
  1631     //  0F 84 03 00 00 00    je          done
  1632     //                  normal_case:
  1633     //  99                   cdq
  1634     //  F7 F9                idiv        rax,ecx
  1635     //                  done:
  1636     //
  1637     emit_opcode(cbuf,0x81); emit_d8(cbuf,0xF8);
  1638     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);
  1639     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x80);                     // cmp rax,80000000h
  1640     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x85);
  1641     emit_opcode(cbuf,0x0B); emit_d8(cbuf,0x00);
  1642     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // jne normal_case
  1643     emit_opcode(cbuf,0x33); emit_d8(cbuf,0xD2);                     // xor rdx,edx
  1644     emit_opcode(cbuf,0x83); emit_d8(cbuf,0xF9); emit_d8(cbuf,0xFF); // cmp rcx,0FFh
  1645     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x84);
  1646     emit_opcode(cbuf,0x03); emit_d8(cbuf,0x00);
  1647     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // je done
  1648     // normal_case:
  1649     emit_opcode(cbuf,0x99);                                         // cdq
  1650     // idiv (note: must be emitted by the user of this rule)
  1651     // normal:
  1652   %}
  1654   // Dense encoding for older common ops
  1655   enc_class Opc_plus(immI opcode, eRegI reg) %{
  1656     emit_opcode(cbuf, $opcode$$constant + $reg$$reg);
  1657   %}
  1660   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
  1661   enc_class OpcSE (immI imm) %{ // Emit primary opcode and set sign-extend bit
  1662     // Check for 8-bit immediate, and set sign extend bit in opcode
  1663     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
  1664       emit_opcode(cbuf, $primary | 0x02);
  1666     else {                          // If 32-bit immediate
  1667       emit_opcode(cbuf, $primary);
  1669   %}
  1671   enc_class OpcSErm (eRegI dst, immI imm) %{    // OpcSEr/m
  1672     // Emit primary opcode and set sign-extend bit
  1673     // Check for 8-bit immediate, and set sign extend bit in opcode
  1674     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
  1675       emit_opcode(cbuf, $primary | 0x02);    }
  1676     else {                          // If 32-bit immediate
  1677       emit_opcode(cbuf, $primary);
  1679     // Emit r/m byte with secondary opcode, after primary opcode.
  1680     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
  1681   %}
  1683   enc_class Con8or32 (immI imm) %{    // Con8or32(storeImmI), 8 or 32 bits
  1684     // Check for 8-bit immediate, and set sign extend bit in opcode
  1685     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
  1686       $$$emit8$imm$$constant;
  1688     else {                          // If 32-bit immediate
  1689       // Output immediate
  1690       $$$emit32$imm$$constant;
  1692   %}
  1694   enc_class Long_OpcSErm_Lo(eRegL dst, immL imm) %{
  1695     // Emit primary opcode and set sign-extend bit
  1696     // Check for 8-bit immediate, and set sign extend bit in opcode
  1697     int con = (int)$imm$$constant; // Throw away top bits
  1698     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);
  1699     // Emit r/m byte with secondary opcode, after primary opcode.
  1700     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
  1701     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);
  1702     else                               emit_d32(cbuf,con);
  1703   %}
  1705   enc_class Long_OpcSErm_Hi(eRegL dst, immL imm) %{
  1706     // Emit primary opcode and set sign-extend bit
  1707     // Check for 8-bit immediate, and set sign extend bit in opcode
  1708     int con = (int)($imm$$constant >> 32); // Throw away bottom bits
  1709     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);
  1710     // Emit r/m byte with tertiary opcode, after primary opcode.
  1711     emit_rm(cbuf, 0x3, $tertiary, HIGH_FROM_LOW($dst$$reg));
  1712     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);
  1713     else                               emit_d32(cbuf,con);
  1714   %}
  1716   enc_class Lbl (label labl) %{ // JMP, CALL
  1717     Label *l = $labl$$label;
  1718     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.insts_size()+4)) : 0);
  1719   %}
  1721   enc_class LblShort (label labl) %{ // JMP, CALL
  1722     Label *l = $labl$$label;
  1723     int disp = l ? (l->loc_pos() - (cbuf.insts_size()+1)) : 0;
  1724     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
  1725     emit_d8(cbuf, disp);
  1726   %}
  1728   enc_class OpcSReg (eRegI dst) %{    // BSWAP
  1729     emit_cc(cbuf, $secondary, $dst$$reg );
  1730   %}
  1732   enc_class bswap_long_bytes(eRegL dst) %{ // BSWAP
  1733     int destlo = $dst$$reg;
  1734     int desthi = HIGH_FROM_LOW(destlo);
  1735     // bswap lo
  1736     emit_opcode(cbuf, 0x0F);
  1737     emit_cc(cbuf, 0xC8, destlo);
  1738     // bswap hi
  1739     emit_opcode(cbuf, 0x0F);
  1740     emit_cc(cbuf, 0xC8, desthi);
  1741     // xchg lo and hi
  1742     emit_opcode(cbuf, 0x87);
  1743     emit_rm(cbuf, 0x3, destlo, desthi);
  1744   %}
  1746   enc_class RegOpc (eRegI div) %{    // IDIV, IMOD, JMP indirect, ...
  1747     emit_rm(cbuf, 0x3, $secondary, $div$$reg );
  1748   %}
  1750   enc_class Jcc (cmpOp cop, label labl) %{    // JCC
  1751     Label *l = $labl$$label;
  1752     $$$emit8$primary;
  1753     emit_cc(cbuf, $secondary, $cop$$cmpcode);
  1754     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.insts_size()+4)) : 0);
  1755   %}
  1757   enc_class JccShort (cmpOp cop, label labl) %{    // JCC
  1758     Label *l = $labl$$label;
  1759     emit_cc(cbuf, $primary, $cop$$cmpcode);
  1760     int disp = l ? (l->loc_pos() - (cbuf.insts_size()+1)) : 0;
  1761     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
  1762     emit_d8(cbuf, disp);
  1763   %}
  1765   enc_class enc_cmov(cmpOp cop ) %{ // CMOV
  1766     $$$emit8$primary;
  1767     emit_cc(cbuf, $secondary, $cop$$cmpcode);
  1768   %}
  1770   enc_class enc_cmov_d(cmpOp cop, regD src ) %{ // CMOV
  1771     int op = 0xDA00 + $cop$$cmpcode + ($src$$reg-1);
  1772     emit_d8(cbuf, op >> 8 );
  1773     emit_d8(cbuf, op & 255);
  1774   %}
  1776   // emulate a CMOV with a conditional branch around a MOV
  1777   enc_class enc_cmov_branch( cmpOp cop, immI brOffs ) %{ // CMOV
  1778     // Invert sense of branch from sense of CMOV
  1779     emit_cc( cbuf, 0x70, ($cop$$cmpcode^1) );
  1780     emit_d8( cbuf, $brOffs$$constant );
  1781   %}
  1783   enc_class enc_PartialSubtypeCheck( ) %{
  1784     Register Redi = as_Register(EDI_enc); // result register
  1785     Register Reax = as_Register(EAX_enc); // super class
  1786     Register Recx = as_Register(ECX_enc); // killed
  1787     Register Resi = as_Register(ESI_enc); // sub class
  1788     Label miss;
  1790     MacroAssembler _masm(&cbuf);
  1791     __ check_klass_subtype_slow_path(Resi, Reax, Recx, Redi,
  1792                                      NULL, &miss,
  1793                                      /*set_cond_codes:*/ true);
  1794     if ($primary) {
  1795       __ xorptr(Redi, Redi);
  1797     __ bind(miss);
  1798   %}
  1800   enc_class FFree_Float_Stack_All %{    // Free_Float_Stack_All
  1801     MacroAssembler masm(&cbuf);
  1802     int start = masm.offset();
  1803     if (UseSSE >= 2) {
  1804       if (VerifyFPU) {
  1805         masm.verify_FPU(0, "must be empty in SSE2+ mode");
  1807     } else {
  1808       // External c_calling_convention expects the FPU stack to be 'clean'.
  1809       // Compiled code leaves it dirty.  Do cleanup now.
  1810       masm.empty_FPU_stack();
  1812     if (sizeof_FFree_Float_Stack_All == -1) {
  1813       sizeof_FFree_Float_Stack_All = masm.offset() - start;
  1814     } else {
  1815       assert(masm.offset() - start == sizeof_FFree_Float_Stack_All, "wrong size");
  1817   %}
  1819   enc_class Verify_FPU_For_Leaf %{
  1820     if( VerifyFPU ) {
  1821       MacroAssembler masm(&cbuf);
  1822       masm.verify_FPU( -3, "Returning from Runtime Leaf call");
  1824   %}
  1826   enc_class Java_To_Runtime (method meth) %{    // CALL Java_To_Runtime, Java_To_Runtime_Leaf
  1827     // This is the instruction starting address for relocation info.
  1828     cbuf.set_insts_mark();
  1829     $$$emit8$primary;
  1830     // CALL directly to the runtime
  1831     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
  1832                 runtime_call_Relocation::spec(), RELOC_IMM32 );
  1834     if (UseSSE >= 2) {
  1835       MacroAssembler _masm(&cbuf);
  1836       BasicType rt = tf()->return_type();
  1838       if ((rt == T_FLOAT || rt == T_DOUBLE) && !return_value_is_used()) {
  1839         // A C runtime call where the return value is unused.  In SSE2+
  1840         // mode the result needs to be removed from the FPU stack.  It's
  1841         // likely that this function call could be removed by the
  1842         // optimizer if the C function is a pure function.
  1843         __ ffree(0);
  1844       } else if (rt == T_FLOAT) {
  1845         __ lea(rsp, Address(rsp, -4));
  1846         __ fstp_s(Address(rsp, 0));
  1847         __ movflt(xmm0, Address(rsp, 0));
  1848         __ lea(rsp, Address(rsp,  4));
  1849       } else if (rt == T_DOUBLE) {
  1850         __ lea(rsp, Address(rsp, -8));
  1851         __ fstp_d(Address(rsp, 0));
  1852         __ movdbl(xmm0, Address(rsp, 0));
  1853         __ lea(rsp, Address(rsp,  8));
  1856   %}
  1859   enc_class pre_call_FPU %{
  1860     // If method sets FPU control word restore it here
  1861     debug_only(int off0 = cbuf.insts_size());
  1862     if( Compile::current()->in_24_bit_fp_mode() ) {
  1863       MacroAssembler masm(&cbuf);
  1864       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
  1866     debug_only(int off1 = cbuf.insts_size());
  1867     assert(off1 - off0 == pre_call_FPU_size(), "correct size prediction");
  1868   %}
  1870   enc_class post_call_FPU %{
  1871     // If method sets FPU control word do it here also
  1872     if( Compile::current()->in_24_bit_fp_mode() ) {
  1873       MacroAssembler masm(&cbuf);
  1874       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
  1876   %}
  1878   enc_class preserve_SP %{
  1879     debug_only(int off0 = cbuf.insts_size());
  1880     MacroAssembler _masm(&cbuf);
  1881     // RBP is preserved across all calls, even compiled calls.
  1882     // Use it to preserve RSP in places where the callee might change the SP.
  1883     __ movptr(rbp_mh_SP_save, rsp);
  1884     debug_only(int off1 = cbuf.insts_size());
  1885     assert(off1 - off0 == preserve_SP_size(), "correct size prediction");
  1886   %}
  1888   enc_class restore_SP %{
  1889     MacroAssembler _masm(&cbuf);
  1890     __ movptr(rsp, rbp_mh_SP_save);
  1891   %}
  1893   enc_class Java_Static_Call (method meth) %{    // JAVA STATIC CALL
  1894     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
  1895     // who we intended to call.
  1896     cbuf.set_insts_mark();
  1897     $$$emit8$primary;
  1898     if ( !_method ) {
  1899       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
  1900                      runtime_call_Relocation::spec(), RELOC_IMM32 );
  1901     } else if(_optimized_virtual) {
  1902       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
  1903                      opt_virtual_call_Relocation::spec(), RELOC_IMM32 );
  1904     } else {
  1905       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
  1906                      static_call_Relocation::spec(), RELOC_IMM32 );
  1908     if( _method ) {  // Emit stub for static call
  1909       emit_java_to_interp(cbuf);
  1911   %}
  1913   enc_class Java_Dynamic_Call (method meth) %{    // JAVA DYNAMIC CALL
  1914     // !!!!!
  1915     // Generate  "Mov EAX,0x00", placeholder instruction to load oop-info
  1916     // emit_call_dynamic_prologue( cbuf );
  1917     cbuf.set_insts_mark();
  1918     emit_opcode(cbuf, 0xB8 + EAX_enc);        // mov    EAX,-1
  1919     emit_d32_reloc(cbuf, (int)Universe::non_oop_word(), oop_Relocation::spec_for_immediate(), RELOC_IMM32);
  1920     address  virtual_call_oop_addr = cbuf.insts_mark();
  1921     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
  1922     // who we intended to call.
  1923     cbuf.set_insts_mark();
  1924     $$$emit8$primary;
  1925     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
  1926                 virtual_call_Relocation::spec(virtual_call_oop_addr), RELOC_IMM32 );
  1927   %}
  1929   enc_class Java_Compiled_Call (method meth) %{    // JAVA COMPILED CALL
  1930     int disp = in_bytes(methodOopDesc::from_compiled_offset());
  1931     assert( -128 <= disp && disp <= 127, "compiled_code_offset isn't small");
  1933     // CALL *[EAX+in_bytes(methodOopDesc::from_compiled_code_entry_point_offset())]
  1934     cbuf.set_insts_mark();
  1935     $$$emit8$primary;
  1936     emit_rm(cbuf, 0x01, $secondary, EAX_enc );  // R/M byte
  1937     emit_d8(cbuf, disp);             // Displacement
  1939   %}
  1941   enc_class Xor_Reg (eRegI dst) %{
  1942     emit_opcode(cbuf, 0x33);
  1943     emit_rm(cbuf, 0x3, $dst$$reg, $dst$$reg);
  1944   %}
  1946 //   Following encoding is no longer used, but may be restored if calling
  1947 //   convention changes significantly.
  1948 //   Became: Xor_Reg(EBP), Java_To_Runtime( labl )
  1949 //
  1950 //   enc_class Java_Interpreter_Call (label labl) %{    // JAVA INTERPRETER CALL
  1951 //     // int ic_reg     = Matcher::inline_cache_reg();
  1952 //     // int ic_encode  = Matcher::_regEncode[ic_reg];
  1953 //     // int imo_reg    = Matcher::interpreter_method_oop_reg();
  1954 //     // int imo_encode = Matcher::_regEncode[imo_reg];
  1955 //
  1956 //     // // Interpreter expects method_oop in EBX, currently a callee-saved register,
  1957 //     // // so we load it immediately before the call
  1958 //     // emit_opcode(cbuf, 0x8B);                     // MOV    imo_reg,ic_reg  # method_oop
  1959 //     // emit_rm(cbuf, 0x03, imo_encode, ic_encode ); // R/M byte
  1960 //
  1961 //     // xor rbp,ebp
  1962 //     emit_opcode(cbuf, 0x33);
  1963 //     emit_rm(cbuf, 0x3, EBP_enc, EBP_enc);
  1964 //
  1965 //     // CALL to interpreter.
  1966 //     cbuf.set_insts_mark();
  1967 //     $$$emit8$primary;
  1968 //     emit_d32_reloc(cbuf, ($labl$$label - (int)(cbuf.insts_end()) - 4),
  1969 //                 runtime_call_Relocation::spec(), RELOC_IMM32 );
  1970 //   %}
  1972   enc_class RegOpcImm (eRegI dst, immI8 shift) %{    // SHL, SAR, SHR
  1973     $$$emit8$primary;
  1974     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
  1975     $$$emit8$shift$$constant;
  1976   %}
  1978   enc_class LdImmI (eRegI dst, immI src) %{    // Load Immediate
  1979     // Load immediate does not have a zero or sign extended version
  1980     // for 8-bit immediates
  1981     emit_opcode(cbuf, 0xB8 + $dst$$reg);
  1982     $$$emit32$src$$constant;
  1983   %}
  1985   enc_class LdImmP (eRegI dst, immI src) %{    // Load Immediate
  1986     // Load immediate does not have a zero or sign extended version
  1987     // for 8-bit immediates
  1988     emit_opcode(cbuf, $primary + $dst$$reg);
  1989     $$$emit32$src$$constant;
  1990   %}
  1992   enc_class LdImmL_Lo( eRegL dst, immL src) %{    // Load Immediate
  1993     // Load immediate does not have a zero or sign extended version
  1994     // for 8-bit immediates
  1995     int dst_enc = $dst$$reg;
  1996     int src_con = $src$$constant & 0x0FFFFFFFFL;
  1997     if (src_con == 0) {
  1998       // xor dst, dst
  1999       emit_opcode(cbuf, 0x33);
  2000       emit_rm(cbuf, 0x3, dst_enc, dst_enc);
  2001     } else {
  2002       emit_opcode(cbuf, $primary + dst_enc);
  2003       emit_d32(cbuf, src_con);
  2005   %}
  2007   enc_class LdImmL_Hi( eRegL dst, immL src) %{    // Load Immediate
  2008     // Load immediate does not have a zero or sign extended version
  2009     // for 8-bit immediates
  2010     int dst_enc = $dst$$reg + 2;
  2011     int src_con = ((julong)($src$$constant)) >> 32;
  2012     if (src_con == 0) {
  2013       // xor dst, dst
  2014       emit_opcode(cbuf, 0x33);
  2015       emit_rm(cbuf, 0x3, dst_enc, dst_enc);
  2016     } else {
  2017       emit_opcode(cbuf, $primary + dst_enc);
  2018       emit_d32(cbuf, src_con);
  2020   %}
  2023   enc_class MovI2X_reg(regX dst, eRegI src) %{
  2024     emit_opcode(cbuf, 0x66 );     // MOVD dst,src
  2025     emit_opcode(cbuf, 0x0F );
  2026     emit_opcode(cbuf, 0x6E );
  2027     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  2028   %}
  2030   enc_class MovX2I_reg(eRegI dst, regX src) %{
  2031     emit_opcode(cbuf, 0x66 );     // MOVD dst,src
  2032     emit_opcode(cbuf, 0x0F );
  2033     emit_opcode(cbuf, 0x7E );
  2034     emit_rm(cbuf, 0x3, $src$$reg, $dst$$reg);
  2035   %}
  2037   enc_class MovL2XD_reg(regXD dst, eRegL src, regXD tmp) %{
  2038     { // MOVD $dst,$src.lo
  2039       emit_opcode(cbuf,0x66);
  2040       emit_opcode(cbuf,0x0F);
  2041       emit_opcode(cbuf,0x6E);
  2042       emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  2044     { // MOVD $tmp,$src.hi
  2045       emit_opcode(cbuf,0x66);
  2046       emit_opcode(cbuf,0x0F);
  2047       emit_opcode(cbuf,0x6E);
  2048       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg));
  2050     { // PUNPCKLDQ $dst,$tmp
  2051       emit_opcode(cbuf,0x66);
  2052       emit_opcode(cbuf,0x0F);
  2053       emit_opcode(cbuf,0x62);
  2054       emit_rm(cbuf, 0x3, $dst$$reg, $tmp$$reg);
  2056   %}
  2058   enc_class MovXD2L_reg(eRegL dst, regXD src, regXD tmp) %{
  2059     { // MOVD $dst.lo,$src
  2060       emit_opcode(cbuf,0x66);
  2061       emit_opcode(cbuf,0x0F);
  2062       emit_opcode(cbuf,0x7E);
  2063       emit_rm(cbuf, 0x3, $src$$reg, $dst$$reg);
  2065     { // PSHUFLW $tmp,$src,0x4E  (01001110b)
  2066       emit_opcode(cbuf,0xF2);
  2067       emit_opcode(cbuf,0x0F);
  2068       emit_opcode(cbuf,0x70);
  2069       emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);
  2070       emit_d8(cbuf, 0x4E);
  2072     { // MOVD $dst.hi,$tmp
  2073       emit_opcode(cbuf,0x66);
  2074       emit_opcode(cbuf,0x0F);
  2075       emit_opcode(cbuf,0x7E);
  2076       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg));
  2078   %}
  2081   // Encode a reg-reg copy.  If it is useless, then empty encoding.
  2082   enc_class enc_Copy( eRegI dst, eRegI src ) %{
  2083     encode_Copy( cbuf, $dst$$reg, $src$$reg );
  2084   %}
  2086   enc_class enc_CopyL_Lo( eRegI dst, eRegL src ) %{
  2087     encode_Copy( cbuf, $dst$$reg, $src$$reg );
  2088   %}
  2090   // Encode xmm reg-reg copy.  If it is useless, then empty encoding.
  2091   enc_class enc_CopyXD( RegXD dst, RegXD src ) %{
  2092     encode_CopyXD( cbuf, $dst$$reg, $src$$reg );
  2093   %}
  2095   enc_class RegReg (eRegI dst, eRegI src) %{    // RegReg(Many)
  2096     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  2097   %}
  2099   enc_class RegReg_Lo(eRegL dst, eRegL src) %{    // RegReg(Many)
  2100     $$$emit8$primary;
  2101     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  2102   %}
  2104   enc_class RegReg_Hi(eRegL dst, eRegL src) %{    // RegReg(Many)
  2105     $$$emit8$secondary;
  2106     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));
  2107   %}
  2109   enc_class RegReg_Lo2(eRegL dst, eRegL src) %{    // RegReg(Many)
  2110     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  2111   %}
  2113   enc_class RegReg_Hi2(eRegL dst, eRegL src) %{    // RegReg(Many)
  2114     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));
  2115   %}
  2117   enc_class RegReg_HiLo( eRegL src, eRegI dst ) %{
  2118     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($src$$reg));
  2119   %}
  2121   enc_class Con32 (immI src) %{    // Con32(storeImmI)
  2122     // Output immediate
  2123     $$$emit32$src$$constant;
  2124   %}
  2126   enc_class Con32F_as_bits(immF src) %{        // storeF_imm
  2127     // Output Float immediate bits
  2128     jfloat jf = $src$$constant;
  2129     int    jf_as_bits = jint_cast( jf );
  2130     emit_d32(cbuf, jf_as_bits);
  2131   %}
  2133   enc_class Con32XF_as_bits(immXF src) %{      // storeX_imm
  2134     // Output Float immediate bits
  2135     jfloat jf = $src$$constant;
  2136     int    jf_as_bits = jint_cast( jf );
  2137     emit_d32(cbuf, jf_as_bits);
  2138   %}
  2140   enc_class Con16 (immI src) %{    // Con16(storeImmI)
  2141     // Output immediate
  2142     $$$emit16$src$$constant;
  2143   %}
  2145   enc_class Con_d32(immI src) %{
  2146     emit_d32(cbuf,$src$$constant);
  2147   %}
  2149   enc_class conmemref (eRegP t1) %{    // Con32(storeImmI)
  2150     // Output immediate memory reference
  2151     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
  2152     emit_d32(cbuf, 0x00);
  2153   %}
  2155   enc_class lock_prefix( ) %{
  2156     if( os::is_MP() )
  2157       emit_opcode(cbuf,0xF0);         // [Lock]
  2158   %}
  2160   // Cmp-xchg long value.
  2161   // Note: we need to swap rbx, and rcx before and after the
  2162   //       cmpxchg8 instruction because the instruction uses
  2163   //       rcx as the high order word of the new value to store but
  2164   //       our register encoding uses rbx,.
  2165   enc_class enc_cmpxchg8(eSIRegP mem_ptr) %{
  2167     // XCHG  rbx,ecx
  2168     emit_opcode(cbuf,0x87);
  2169     emit_opcode(cbuf,0xD9);
  2170     // [Lock]
  2171     if( os::is_MP() )
  2172       emit_opcode(cbuf,0xF0);
  2173     // CMPXCHG8 [Eptr]
  2174     emit_opcode(cbuf,0x0F);
  2175     emit_opcode(cbuf,0xC7);
  2176     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );
  2177     // XCHG  rbx,ecx
  2178     emit_opcode(cbuf,0x87);
  2179     emit_opcode(cbuf,0xD9);
  2180   %}
  2182   enc_class enc_cmpxchg(eSIRegP mem_ptr) %{
  2183     // [Lock]
  2184     if( os::is_MP() )
  2185       emit_opcode(cbuf,0xF0);
  2187     // CMPXCHG [Eptr]
  2188     emit_opcode(cbuf,0x0F);
  2189     emit_opcode(cbuf,0xB1);
  2190     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );
  2191   %}
  2193   enc_class enc_flags_ne_to_boolean( iRegI res ) %{
  2194     int res_encoding = $res$$reg;
  2196     // MOV  res,0
  2197     emit_opcode( cbuf, 0xB8 + res_encoding);
  2198     emit_d32( cbuf, 0 );
  2199     // JNE,s  fail
  2200     emit_opcode(cbuf,0x75);
  2201     emit_d8(cbuf, 5 );
  2202     // MOV  res,1
  2203     emit_opcode( cbuf, 0xB8 + res_encoding);
  2204     emit_d32( cbuf, 1 );
  2205     // fail:
  2206   %}
  2208   enc_class set_instruction_start( ) %{
  2209     cbuf.set_insts_mark();            // Mark start of opcode for reloc info in mem operand
  2210   %}
  2212   enc_class RegMem (eRegI ereg, memory mem) %{    // emit_reg_mem
  2213     int reg_encoding = $ereg$$reg;
  2214     int base  = $mem$$base;
  2215     int index = $mem$$index;
  2216     int scale = $mem$$scale;
  2217     int displace = $mem$$disp;
  2218     bool disp_is_oop = $mem->disp_is_oop();
  2219     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
  2220   %}
  2222   enc_class RegMem_Hi(eRegL ereg, memory mem) %{    // emit_reg_mem
  2223     int reg_encoding = HIGH_FROM_LOW($ereg$$reg);  // Hi register of pair, computed from lo
  2224     int base  = $mem$$base;
  2225     int index = $mem$$index;
  2226     int scale = $mem$$scale;
  2227     int displace = $mem$$disp + 4;      // Offset is 4 further in memory
  2228     assert( !$mem->disp_is_oop(), "Cannot add 4 to oop" );
  2229     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, false/*disp_is_oop*/);
  2230   %}
  2232   enc_class move_long_small_shift( eRegL dst, immI_1_31 cnt ) %{
  2233     int r1, r2;
  2234     if( $tertiary == 0xA4 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }
  2235     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }
  2236     emit_opcode(cbuf,0x0F);
  2237     emit_opcode(cbuf,$tertiary);
  2238     emit_rm(cbuf, 0x3, r1, r2);
  2239     emit_d8(cbuf,$cnt$$constant);
  2240     emit_d8(cbuf,$primary);
  2241     emit_rm(cbuf, 0x3, $secondary, r1);
  2242     emit_d8(cbuf,$cnt$$constant);
  2243   %}
  2245   enc_class move_long_big_shift_sign( eRegL dst, immI_32_63 cnt ) %{
  2246     emit_opcode( cbuf, 0x8B ); // Move
  2247     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));
  2248     if( $cnt$$constant > 32 ) { // Shift, if not by zero
  2249       emit_d8(cbuf,$primary);
  2250       emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
  2251       emit_d8(cbuf,$cnt$$constant-32);
  2253     emit_d8(cbuf,$primary);
  2254     emit_rm(cbuf, 0x3, $secondary, HIGH_FROM_LOW($dst$$reg));
  2255     emit_d8(cbuf,31);
  2256   %}
  2258   enc_class move_long_big_shift_clr( eRegL dst, immI_32_63 cnt ) %{
  2259     int r1, r2;
  2260     if( $secondary == 0x5 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }
  2261     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }
  2263     emit_opcode( cbuf, 0x8B ); // Move r1,r2
  2264     emit_rm(cbuf, 0x3, r1, r2);
  2265     if( $cnt$$constant > 32 ) { // Shift, if not by zero
  2266       emit_opcode(cbuf,$primary);
  2267       emit_rm(cbuf, 0x3, $secondary, r1);
  2268       emit_d8(cbuf,$cnt$$constant-32);
  2270     emit_opcode(cbuf,0x33);  // XOR r2,r2
  2271     emit_rm(cbuf, 0x3, r2, r2);
  2272   %}
  2274   // Clone of RegMem but accepts an extra parameter to access each
  2275   // half of a double in memory; it never needs relocation info.
  2276   enc_class Mov_MemD_half_to_Reg (immI opcode, memory mem, immI disp_for_half, eRegI rm_reg) %{
  2277     emit_opcode(cbuf,$opcode$$constant);
  2278     int reg_encoding = $rm_reg$$reg;
  2279     int base     = $mem$$base;
  2280     int index    = $mem$$index;
  2281     int scale    = $mem$$scale;
  2282     int displace = $mem$$disp + $disp_for_half$$constant;
  2283     bool disp_is_oop = false;
  2284     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
  2285   %}
  2287   // !!!!! Special Custom Code used by MemMove, and stack access instructions !!!!!
  2288   //
  2289   // Clone of RegMem except the RM-byte's reg/opcode field is an ADLC-time constant
  2290   // and it never needs relocation information.
  2291   // Frequently used to move data between FPU's Stack Top and memory.
  2292   enc_class RMopc_Mem_no_oop (immI rm_opcode, memory mem) %{
  2293     int rm_byte_opcode = $rm_opcode$$constant;
  2294     int base     = $mem$$base;
  2295     int index    = $mem$$index;
  2296     int scale    = $mem$$scale;
  2297     int displace = $mem$$disp;
  2298     assert( !$mem->disp_is_oop(), "No oops here because no relo info allowed" );
  2299     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, false);
  2300   %}
  2302   enc_class RMopc_Mem (immI rm_opcode, memory mem) %{
  2303     int rm_byte_opcode = $rm_opcode$$constant;
  2304     int base     = $mem$$base;
  2305     int index    = $mem$$index;
  2306     int scale    = $mem$$scale;
  2307     int displace = $mem$$disp;
  2308     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
  2309     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_is_oop);
  2310   %}
  2312   enc_class RegLea (eRegI dst, eRegI src0, immI src1 ) %{    // emit_reg_lea
  2313     int reg_encoding = $dst$$reg;
  2314     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
  2315     int index        = 0x04;            // 0x04 indicates no index
  2316     int scale        = 0x00;            // 0x00 indicates no scale
  2317     int displace     = $src1$$constant; // 0x00 indicates no displacement
  2318     bool disp_is_oop = false;
  2319     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
  2320   %}
  2322   enc_class min_enc (eRegI dst, eRegI src) %{    // MIN
  2323     // Compare dst,src
  2324     emit_opcode(cbuf,0x3B);
  2325     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  2326     // jmp dst < src around move
  2327     emit_opcode(cbuf,0x7C);
  2328     emit_d8(cbuf,2);
  2329     // move dst,src
  2330     emit_opcode(cbuf,0x8B);
  2331     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  2332   %}
  2334   enc_class max_enc (eRegI dst, eRegI src) %{    // MAX
  2335     // Compare dst,src
  2336     emit_opcode(cbuf,0x3B);
  2337     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  2338     // jmp dst > src around move
  2339     emit_opcode(cbuf,0x7F);
  2340     emit_d8(cbuf,2);
  2341     // move dst,src
  2342     emit_opcode(cbuf,0x8B);
  2343     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  2344   %}
  2346   enc_class enc_FP_store(memory mem, regD src) %{
  2347     // If src is FPR1, we can just FST to store it.
  2348     // Else we need to FLD it to FPR1, then FSTP to store/pop it.
  2349     int reg_encoding = 0x2; // Just store
  2350     int base  = $mem$$base;
  2351     int index = $mem$$index;
  2352     int scale = $mem$$scale;
  2353     int displace = $mem$$disp;
  2354     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
  2355     if( $src$$reg != FPR1L_enc ) {
  2356       reg_encoding = 0x3;  // Store & pop
  2357       emit_opcode( cbuf, 0xD9 ); // FLD (i.e., push it)
  2358       emit_d8( cbuf, 0xC0-1+$src$$reg );
  2360     cbuf.set_insts_mark();       // Mark start of opcode for reloc info in mem operand
  2361     emit_opcode(cbuf,$primary);
  2362     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
  2363   %}
  2365   enc_class neg_reg(eRegI dst) %{
  2366     // NEG $dst
  2367     emit_opcode(cbuf,0xF7);
  2368     emit_rm(cbuf, 0x3, 0x03, $dst$$reg );
  2369   %}
  2371   enc_class setLT_reg(eCXRegI dst) %{
  2372     // SETLT $dst
  2373     emit_opcode(cbuf,0x0F);
  2374     emit_opcode(cbuf,0x9C);
  2375     emit_rm( cbuf, 0x3, 0x4, $dst$$reg );
  2376   %}
  2378   enc_class enc_cmpLTP(ncxRegI p, ncxRegI q, ncxRegI y, eCXRegI tmp) %{    // cadd_cmpLT
  2379     int tmpReg = $tmp$$reg;
  2381     // SUB $p,$q
  2382     emit_opcode(cbuf,0x2B);
  2383     emit_rm(cbuf, 0x3, $p$$reg, $q$$reg);
  2384     // SBB $tmp,$tmp
  2385     emit_opcode(cbuf,0x1B);
  2386     emit_rm(cbuf, 0x3, tmpReg, tmpReg);
  2387     // AND $tmp,$y
  2388     emit_opcode(cbuf,0x23);
  2389     emit_rm(cbuf, 0x3, tmpReg, $y$$reg);
  2390     // ADD $p,$tmp
  2391     emit_opcode(cbuf,0x03);
  2392     emit_rm(cbuf, 0x3, $p$$reg, tmpReg);
  2393   %}
  2395   enc_class enc_cmpLTP_mem(eRegI p, eRegI q, memory mem, eCXRegI tmp) %{    // cadd_cmpLT
  2396     int tmpReg = $tmp$$reg;
  2398     // SUB $p,$q
  2399     emit_opcode(cbuf,0x2B);
  2400     emit_rm(cbuf, 0x3, $p$$reg, $q$$reg);
  2401     // SBB $tmp,$tmp
  2402     emit_opcode(cbuf,0x1B);
  2403     emit_rm(cbuf, 0x3, tmpReg, tmpReg);
  2404     // AND $tmp,$y
  2405     cbuf.set_insts_mark();       // Mark start of opcode for reloc info in mem operand
  2406     emit_opcode(cbuf,0x23);
  2407     int reg_encoding = tmpReg;
  2408     int base  = $mem$$base;
  2409     int index = $mem$$index;
  2410     int scale = $mem$$scale;
  2411     int displace = $mem$$disp;
  2412     bool disp_is_oop = $mem->disp_is_oop();
  2413     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
  2414     // ADD $p,$tmp
  2415     emit_opcode(cbuf,0x03);
  2416     emit_rm(cbuf, 0x3, $p$$reg, tmpReg);
  2417   %}
  2419   enc_class shift_left_long( eRegL dst, eCXRegI shift ) %{
  2420     // TEST shift,32
  2421     emit_opcode(cbuf,0xF7);
  2422     emit_rm(cbuf, 0x3, 0, ECX_enc);
  2423     emit_d32(cbuf,0x20);
  2424     // JEQ,s small
  2425     emit_opcode(cbuf, 0x74);
  2426     emit_d8(cbuf, 0x04);
  2427     // MOV    $dst.hi,$dst.lo
  2428     emit_opcode( cbuf, 0x8B );
  2429     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg );
  2430     // CLR    $dst.lo
  2431     emit_opcode(cbuf, 0x33);
  2432     emit_rm(cbuf, 0x3, $dst$$reg, $dst$$reg);
  2433 // small:
  2434     // SHLD   $dst.hi,$dst.lo,$shift
  2435     emit_opcode(cbuf,0x0F);
  2436     emit_opcode(cbuf,0xA5);
  2437     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));
  2438     // SHL    $dst.lo,$shift"
  2439     emit_opcode(cbuf,0xD3);
  2440     emit_rm(cbuf, 0x3, 0x4, $dst$$reg );
  2441   %}
  2443   enc_class shift_right_long( eRegL dst, eCXRegI shift ) %{
  2444     // TEST shift,32
  2445     emit_opcode(cbuf,0xF7);
  2446     emit_rm(cbuf, 0x3, 0, ECX_enc);
  2447     emit_d32(cbuf,0x20);
  2448     // JEQ,s small
  2449     emit_opcode(cbuf, 0x74);
  2450     emit_d8(cbuf, 0x04);
  2451     // MOV    $dst.lo,$dst.hi
  2452     emit_opcode( cbuf, 0x8B );
  2453     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );
  2454     // CLR    $dst.hi
  2455     emit_opcode(cbuf, 0x33);
  2456     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($dst$$reg));
  2457 // small:
  2458     // SHRD   $dst.lo,$dst.hi,$shift
  2459     emit_opcode(cbuf,0x0F);
  2460     emit_opcode(cbuf,0xAD);
  2461     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);
  2462     // SHR    $dst.hi,$shift"
  2463     emit_opcode(cbuf,0xD3);
  2464     emit_rm(cbuf, 0x3, 0x5, HIGH_FROM_LOW($dst$$reg) );
  2465   %}
  2467   enc_class shift_right_arith_long( eRegL dst, eCXRegI shift ) %{
  2468     // TEST shift,32
  2469     emit_opcode(cbuf,0xF7);
  2470     emit_rm(cbuf, 0x3, 0, ECX_enc);
  2471     emit_d32(cbuf,0x20);
  2472     // JEQ,s small
  2473     emit_opcode(cbuf, 0x74);
  2474     emit_d8(cbuf, 0x05);
  2475     // MOV    $dst.lo,$dst.hi
  2476     emit_opcode( cbuf, 0x8B );
  2477     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );
  2478     // SAR    $dst.hi,31
  2479     emit_opcode(cbuf, 0xC1);
  2480     emit_rm(cbuf, 0x3, 7, HIGH_FROM_LOW($dst$$reg) );
  2481     emit_d8(cbuf, 0x1F );
  2482 // small:
  2483     // SHRD   $dst.lo,$dst.hi,$shift
  2484     emit_opcode(cbuf,0x0F);
  2485     emit_opcode(cbuf,0xAD);
  2486     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);
  2487     // SAR    $dst.hi,$shift"
  2488     emit_opcode(cbuf,0xD3);
  2489     emit_rm(cbuf, 0x3, 0x7, HIGH_FROM_LOW($dst$$reg) );
  2490   %}
  2493   // ----------------- Encodings for floating point unit -----------------
  2494   // May leave result in FPU-TOS or FPU reg depending on opcodes
  2495   enc_class OpcReg_F (regF src) %{    // FMUL, FDIV
  2496     $$$emit8$primary;
  2497     emit_rm(cbuf, 0x3, $secondary, $src$$reg );
  2498   %}
  2500   // Pop argument in FPR0 with FSTP ST(0)
  2501   enc_class PopFPU() %{
  2502     emit_opcode( cbuf, 0xDD );
  2503     emit_d8( cbuf, 0xD8 );
  2504   %}
  2506   // !!!!! equivalent to Pop_Reg_F
  2507   enc_class Pop_Reg_D( regD dst ) %{
  2508     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)
  2509     emit_d8( cbuf, 0xD8+$dst$$reg );
  2510   %}
  2512   enc_class Push_Reg_D( regD dst ) %{
  2513     emit_opcode( cbuf, 0xD9 );
  2514     emit_d8( cbuf, 0xC0-1+$dst$$reg );   // FLD ST(i-1)
  2515   %}
  2517   enc_class strictfp_bias1( regD dst ) %{
  2518     emit_opcode( cbuf, 0xDB );           // FLD m80real
  2519     emit_opcode( cbuf, 0x2D );
  2520     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias1() );
  2521     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0
  2522     emit_opcode( cbuf, 0xC8+$dst$$reg );
  2523   %}
  2525   enc_class strictfp_bias2( regD dst ) %{
  2526     emit_opcode( cbuf, 0xDB );           // FLD m80real
  2527     emit_opcode( cbuf, 0x2D );
  2528     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias2() );
  2529     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0
  2530     emit_opcode( cbuf, 0xC8+$dst$$reg );
  2531   %}
  2533   // Special case for moving an integer register to a stack slot.
  2534   enc_class OpcPRegSS( stackSlotI dst, eRegI src ) %{ // RegSS
  2535     store_to_stackslot( cbuf, $primary, $src$$reg, $dst$$disp );
  2536   %}
  2538   // Special case for moving a register to a stack slot.
  2539   enc_class RegSS( stackSlotI dst, eRegI src ) %{ // RegSS
  2540     // Opcode already emitted
  2541     emit_rm( cbuf, 0x02, $src$$reg, ESP_enc );   // R/M byte
  2542     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);          // SIB byte
  2543     emit_d32(cbuf, $dst$$disp);   // Displacement
  2544   %}
  2546   // Push the integer in stackSlot 'src' onto FP-stack
  2547   enc_class Push_Mem_I( memory src ) %{    // FILD   [ESP+src]
  2548     store_to_stackslot( cbuf, $primary, $secondary, $src$$disp );
  2549   %}
  2551   // Push the float in stackSlot 'src' onto FP-stack
  2552   enc_class Push_Mem_F( memory src ) %{    // FLD_S   [ESP+src]
  2553     store_to_stackslot( cbuf, 0xD9, 0x00, $src$$disp );
  2554   %}
  2556   // Push the double in stackSlot 'src' onto FP-stack
  2557   enc_class Push_Mem_D( memory src ) %{    // FLD_D   [ESP+src]
  2558     store_to_stackslot( cbuf, 0xDD, 0x00, $src$$disp );
  2559   %}
  2561   // Push FPU's TOS float to a stack-slot, and pop FPU-stack
  2562   enc_class Pop_Mem_F( stackSlotF dst ) %{ // FSTP_S [ESP+dst]
  2563     store_to_stackslot( cbuf, 0xD9, 0x03, $dst$$disp );
  2564   %}
  2566   // Same as Pop_Mem_F except for opcode
  2567   // Push FPU's TOS double to a stack-slot, and pop FPU-stack
  2568   enc_class Pop_Mem_D( stackSlotD dst ) %{ // FSTP_D [ESP+dst]
  2569     store_to_stackslot( cbuf, 0xDD, 0x03, $dst$$disp );
  2570   %}
  2572   enc_class Pop_Reg_F( regF dst ) %{
  2573     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)
  2574     emit_d8( cbuf, 0xD8+$dst$$reg );
  2575   %}
  2577   enc_class Push_Reg_F( regF dst ) %{
  2578     emit_opcode( cbuf, 0xD9 );           // FLD    ST(i-1)
  2579     emit_d8( cbuf, 0xC0-1+$dst$$reg );
  2580   %}
  2582   // Push FPU's float to a stack-slot, and pop FPU-stack
  2583   enc_class Pop_Mem_Reg_F( stackSlotF dst, regF src ) %{
  2584     int pop = 0x02;
  2585     if ($src$$reg != FPR1L_enc) {
  2586       emit_opcode( cbuf, 0xD9 );         // FLD    ST(i-1)
  2587       emit_d8( cbuf, 0xC0-1+$src$$reg );
  2588       pop = 0x03;
  2590     store_to_stackslot( cbuf, 0xD9, pop, $dst$$disp ); // FST<P>_S  [ESP+dst]
  2591   %}
  2593   // Push FPU's double to a stack-slot, and pop FPU-stack
  2594   enc_class Pop_Mem_Reg_D( stackSlotD dst, regD src ) %{
  2595     int pop = 0x02;
  2596     if ($src$$reg != FPR1L_enc) {
  2597       emit_opcode( cbuf, 0xD9 );         // FLD    ST(i-1)
  2598       emit_d8( cbuf, 0xC0-1+$src$$reg );
  2599       pop = 0x03;
  2601     store_to_stackslot( cbuf, 0xDD, pop, $dst$$disp ); // FST<P>_D  [ESP+dst]
  2602   %}
  2604   // Push FPU's double to a FPU-stack-slot, and pop FPU-stack
  2605   enc_class Pop_Reg_Reg_D( regD dst, regF src ) %{
  2606     int pop = 0xD0 - 1; // -1 since we skip FLD
  2607     if ($src$$reg != FPR1L_enc) {
  2608       emit_opcode( cbuf, 0xD9 );         // FLD    ST(src-1)
  2609       emit_d8( cbuf, 0xC0-1+$src$$reg );
  2610       pop = 0xD8;
  2612     emit_opcode( cbuf, 0xDD );
  2613     emit_d8( cbuf, pop+$dst$$reg );      // FST<P> ST(i)
  2614   %}
  2617   enc_class Mul_Add_F( regF dst, regF src, regF src1, regF src2 ) %{
  2618     MacroAssembler masm(&cbuf);
  2619     masm.fld_s(  $src1$$reg-1);   // nothing at TOS, load TOS from src1.reg
  2620     masm.fmul(   $src2$$reg+0);   // value at TOS
  2621     masm.fadd(   $src$$reg+0);    // value at TOS
  2622     masm.fstp_d( $dst$$reg+0);    // value at TOS, popped off after store
  2623   %}
  2626   enc_class Push_Reg_Mod_D( regD dst, regD src) %{
  2627     // load dst in FPR0
  2628     emit_opcode( cbuf, 0xD9 );
  2629     emit_d8( cbuf, 0xC0-1+$dst$$reg );
  2630     if ($src$$reg != FPR1L_enc) {
  2631       // fincstp
  2632       emit_opcode (cbuf, 0xD9);
  2633       emit_opcode (cbuf, 0xF7);
  2634       // swap src with FPR1:
  2635       // FXCH FPR1 with src
  2636       emit_opcode(cbuf, 0xD9);
  2637       emit_d8(cbuf, 0xC8-1+$src$$reg );
  2638       // fdecstp
  2639       emit_opcode (cbuf, 0xD9);
  2640       emit_opcode (cbuf, 0xF6);
  2642   %}
  2644   enc_class Push_ModD_encoding( regXD src0, regXD src1) %{
  2645     // Allocate a word
  2646     emit_opcode(cbuf,0x83);            // SUB ESP,8
  2647     emit_opcode(cbuf,0xEC);
  2648     emit_d8(cbuf,0x08);
  2650     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], src1
  2651     emit_opcode  (cbuf, 0x0F );
  2652     emit_opcode  (cbuf, 0x11 );
  2653     encode_RegMem(cbuf, $src1$$reg, ESP_enc, 0x4, 0, 0, false);
  2655     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
  2656     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  2658     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], src0
  2659     emit_opcode  (cbuf, 0x0F );
  2660     emit_opcode  (cbuf, 0x11 );
  2661     encode_RegMem(cbuf, $src0$$reg, ESP_enc, 0x4, 0, 0, false);
  2663     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
  2664     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  2666   %}
  2668   enc_class Push_ModX_encoding( regX src0, regX src1) %{
  2669     // Allocate a word
  2670     emit_opcode(cbuf,0x83);            // SUB ESP,4
  2671     emit_opcode(cbuf,0xEC);
  2672     emit_d8(cbuf,0x04);
  2674     emit_opcode  (cbuf, 0xF3 );     // MOVSS [ESP], src1
  2675     emit_opcode  (cbuf, 0x0F );
  2676     emit_opcode  (cbuf, 0x11 );
  2677     encode_RegMem(cbuf, $src1$$reg, ESP_enc, 0x4, 0, 0, false);
  2679     emit_opcode(cbuf,0xD9 );      // FLD [ESP]
  2680     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  2682     emit_opcode  (cbuf, 0xF3 );     // MOVSS [ESP], src0
  2683     emit_opcode  (cbuf, 0x0F );
  2684     emit_opcode  (cbuf, 0x11 );
  2685     encode_RegMem(cbuf, $src0$$reg, ESP_enc, 0x4, 0, 0, false);
  2687     emit_opcode(cbuf,0xD9 );      // FLD [ESP]
  2688     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  2690   %}
  2692   enc_class Push_ResultXD(regXD dst) %{
  2693     store_to_stackslot( cbuf, 0xDD, 0x03, 0 ); //FSTP [ESP]
  2695     // UseXmmLoadAndClearUpper ? movsd dst,[esp] : movlpd dst,[esp]
  2696     emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
  2697     emit_opcode  (cbuf, 0x0F );
  2698     emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);
  2699     encode_RegMem(cbuf, $dst$$reg, ESP_enc, 0x4, 0, 0, false);
  2701     emit_opcode(cbuf,0x83);    // ADD ESP,8
  2702     emit_opcode(cbuf,0xC4);
  2703     emit_d8(cbuf,0x08);
  2704   %}
  2706   enc_class Push_ResultX(regX dst, immI d8) %{
  2707     store_to_stackslot( cbuf, 0xD9, 0x03, 0 ); //FSTP_S [ESP]
  2709     emit_opcode  (cbuf, 0xF3 );     // MOVSS dst(xmm), [ESP]
  2710     emit_opcode  (cbuf, 0x0F );
  2711     emit_opcode  (cbuf, 0x10 );
  2712     encode_RegMem(cbuf, $dst$$reg, ESP_enc, 0x4, 0, 0, false);
  2714     emit_opcode(cbuf,0x83);    // ADD ESP,d8 (4 or 8)
  2715     emit_opcode(cbuf,0xC4);
  2716     emit_d8(cbuf,$d8$$constant);
  2717   %}
  2719   enc_class Push_SrcXD(regXD src) %{
  2720     // Allocate a word
  2721     emit_opcode(cbuf,0x83);            // SUB ESP,8
  2722     emit_opcode(cbuf,0xEC);
  2723     emit_d8(cbuf,0x08);
  2725     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], src
  2726     emit_opcode  (cbuf, 0x0F );
  2727     emit_opcode  (cbuf, 0x11 );
  2728     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
  2730     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
  2731     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  2732   %}
  2734   enc_class push_stack_temp_qword() %{
  2735     emit_opcode(cbuf,0x83);     // SUB ESP,8
  2736     emit_opcode(cbuf,0xEC);
  2737     emit_d8    (cbuf,0x08);
  2738   %}
  2740   enc_class pop_stack_temp_qword() %{
  2741     emit_opcode(cbuf,0x83);     // ADD ESP,8
  2742     emit_opcode(cbuf,0xC4);
  2743     emit_d8    (cbuf,0x08);
  2744   %}
  2746   enc_class push_xmm_to_fpr1( regXD xmm_src ) %{
  2747     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], xmm_src
  2748     emit_opcode  (cbuf, 0x0F );
  2749     emit_opcode  (cbuf, 0x11 );
  2750     encode_RegMem(cbuf, $xmm_src$$reg, ESP_enc, 0x4, 0, 0, false);
  2752     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
  2753     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  2754   %}
  2756   // Compute X^Y using Intel's fast hardware instructions, if possible.
  2757   // Otherwise return a NaN.
  2758   enc_class pow_exp_core_encoding %{
  2759     // FPR1 holds Y*ln2(X).  Compute FPR1 = 2^(Y*ln2(X))
  2760     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xC0);  // fdup = fld st(0)          Q       Q
  2761     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xFC);  // frndint               int(Q)      Q
  2762     emit_opcode(cbuf,0xDC); emit_opcode(cbuf,0xE9);  // fsub st(1) -= st(0);  int(Q) frac(Q)
  2763     emit_opcode(cbuf,0xDB);                          // FISTP [ESP]           frac(Q)
  2764     emit_opcode(cbuf,0x1C);
  2765     emit_d8(cbuf,0x24);
  2766     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xF0);  // f2xm1                 2^frac(Q)-1
  2767     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xE8);  // fld1                  1 2^frac(Q)-1
  2768     emit_opcode(cbuf,0xDE); emit_opcode(cbuf,0xC1);  // faddp                 2^frac(Q)
  2769     emit_opcode(cbuf,0x8B);                          // mov rax,[esp+0]=int(Q)
  2770     encode_RegMem(cbuf, EAX_enc, ESP_enc, 0x4, 0, 0, false);
  2771     emit_opcode(cbuf,0xC7);                          // mov rcx,0xFFFFF800 - overflow mask
  2772     emit_rm(cbuf, 0x3, 0x0, ECX_enc);
  2773     emit_d32(cbuf,0xFFFFF800);
  2774     emit_opcode(cbuf,0x81);                          // add rax,1023 - the double exponent bias
  2775     emit_rm(cbuf, 0x3, 0x0, EAX_enc);
  2776     emit_d32(cbuf,1023);
  2777     emit_opcode(cbuf,0x8B);                          // mov rbx,eax
  2778     emit_rm(cbuf, 0x3, EBX_enc, EAX_enc);
  2779     emit_opcode(cbuf,0xC1);                          // shl rax,20 - Slide to exponent position
  2780     emit_rm(cbuf,0x3,0x4,EAX_enc);
  2781     emit_d8(cbuf,20);
  2782     emit_opcode(cbuf,0x85);                          // test rbx,ecx - check for overflow
  2783     emit_rm(cbuf, 0x3, EBX_enc, ECX_enc);
  2784     emit_opcode(cbuf,0x0F); emit_opcode(cbuf,0x45);  // CMOVne rax,ecx - overflow; stuff NAN into EAX
  2785     emit_rm(cbuf, 0x3, EAX_enc, ECX_enc);
  2786     emit_opcode(cbuf,0x89);                          // mov [esp+4],eax - Store as part of double word
  2787     encode_RegMem(cbuf, EAX_enc, ESP_enc, 0x4, 0, 4, false);
  2788     emit_opcode(cbuf,0xC7);                          // mov [esp+0],0   - [ESP] = (double)(1<<int(Q)) = 2^int(Q)
  2789     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  2790     emit_d32(cbuf,0);
  2791     emit_opcode(cbuf,0xDC);                          // fmul dword st(0),[esp+0]; FPR1 = 2^int(Q)*2^frac(Q) = 2^Q
  2792     encode_RegMem(cbuf, 0x1, ESP_enc, 0x4, 0, 0, false);
  2793   %}
  2795 //   enc_class Pop_Reg_Mod_D( regD dst, regD src)
  2796 //   was replaced by Push_Result_Mod_D followed by Pop_Reg_X() or Pop_Mem_X()
  2798   enc_class Push_Result_Mod_D( regD src) %{
  2799     if ($src$$reg != FPR1L_enc) {
  2800       // fincstp
  2801       emit_opcode (cbuf, 0xD9);
  2802       emit_opcode (cbuf, 0xF7);
  2803       // FXCH FPR1 with src
  2804       emit_opcode(cbuf, 0xD9);
  2805       emit_d8(cbuf, 0xC8-1+$src$$reg );
  2806       // fdecstp
  2807       emit_opcode (cbuf, 0xD9);
  2808       emit_opcode (cbuf, 0xF6);
  2810     // // following asm replaced with Pop_Reg_F or Pop_Mem_F
  2811     // // FSTP   FPR$dst$$reg
  2812     // emit_opcode( cbuf, 0xDD );
  2813     // emit_d8( cbuf, 0xD8+$dst$$reg );
  2814   %}
  2816   enc_class fnstsw_sahf_skip_parity() %{
  2817     // fnstsw ax
  2818     emit_opcode( cbuf, 0xDF );
  2819     emit_opcode( cbuf, 0xE0 );
  2820     // sahf
  2821     emit_opcode( cbuf, 0x9E );
  2822     // jnp  ::skip
  2823     emit_opcode( cbuf, 0x7B );
  2824     emit_opcode( cbuf, 0x05 );
  2825   %}
  2827   enc_class emitModD() %{
  2828     // fprem must be iterative
  2829     // :: loop
  2830     // fprem
  2831     emit_opcode( cbuf, 0xD9 );
  2832     emit_opcode( cbuf, 0xF8 );
  2833     // wait
  2834     emit_opcode( cbuf, 0x9b );
  2835     // fnstsw ax
  2836     emit_opcode( cbuf, 0xDF );
  2837     emit_opcode( cbuf, 0xE0 );
  2838     // sahf
  2839     emit_opcode( cbuf, 0x9E );
  2840     // jp  ::loop
  2841     emit_opcode( cbuf, 0x0F );
  2842     emit_opcode( cbuf, 0x8A );
  2843     emit_opcode( cbuf, 0xF4 );
  2844     emit_opcode( cbuf, 0xFF );
  2845     emit_opcode( cbuf, 0xFF );
  2846     emit_opcode( cbuf, 0xFF );
  2847   %}
  2849   enc_class fpu_flags() %{
  2850     // fnstsw_ax
  2851     emit_opcode( cbuf, 0xDF);
  2852     emit_opcode( cbuf, 0xE0);
  2853     // test ax,0x0400
  2854     emit_opcode( cbuf, 0x66 );   // operand-size prefix for 16-bit immediate
  2855     emit_opcode( cbuf, 0xA9 );
  2856     emit_d16   ( cbuf, 0x0400 );
  2857     // // // This sequence works, but stalls for 12-16 cycles on PPro
  2858     // // test rax,0x0400
  2859     // emit_opcode( cbuf, 0xA9 );
  2860     // emit_d32   ( cbuf, 0x00000400 );
  2861     //
  2862     // jz exit (no unordered comparison)
  2863     emit_opcode( cbuf, 0x74 );
  2864     emit_d8    ( cbuf, 0x02 );
  2865     // mov ah,1 - treat as LT case (set carry flag)
  2866     emit_opcode( cbuf, 0xB4 );
  2867     emit_d8    ( cbuf, 0x01 );
  2868     // sahf
  2869     emit_opcode( cbuf, 0x9E);
  2870   %}
  2872   enc_class cmpF_P6_fixup() %{
  2873     // Fixup the integer flags in case comparison involved a NaN
  2874     //
  2875     // JNP exit (no unordered comparison, P-flag is set by NaN)
  2876     emit_opcode( cbuf, 0x7B );
  2877     emit_d8    ( cbuf, 0x03 );
  2878     // MOV AH,1 - treat as LT case (set carry flag)
  2879     emit_opcode( cbuf, 0xB4 );
  2880     emit_d8    ( cbuf, 0x01 );
  2881     // SAHF
  2882     emit_opcode( cbuf, 0x9E);
  2883     // NOP     // target for branch to avoid branch to branch
  2884     emit_opcode( cbuf, 0x90);
  2885   %}
  2887 //     fnstsw_ax();
  2888 //     sahf();
  2889 //     movl(dst, nan_result);
  2890 //     jcc(Assembler::parity, exit);
  2891 //     movl(dst, less_result);
  2892 //     jcc(Assembler::below, exit);
  2893 //     movl(dst, equal_result);
  2894 //     jcc(Assembler::equal, exit);
  2895 //     movl(dst, greater_result);
  2897 // less_result     =  1;
  2898 // greater_result  = -1;
  2899 // equal_result    = 0;
  2900 // nan_result      = -1;
  2902   enc_class CmpF_Result(eRegI dst) %{
  2903     // fnstsw_ax();
  2904     emit_opcode( cbuf, 0xDF);
  2905     emit_opcode( cbuf, 0xE0);
  2906     // sahf
  2907     emit_opcode( cbuf, 0x9E);
  2908     // movl(dst, nan_result);
  2909     emit_opcode( cbuf, 0xB8 + $dst$$reg);
  2910     emit_d32( cbuf, -1 );
  2911     // jcc(Assembler::parity, exit);
  2912     emit_opcode( cbuf, 0x7A );
  2913     emit_d8    ( cbuf, 0x13 );
  2914     // movl(dst, less_result);
  2915     emit_opcode( cbuf, 0xB8 + $dst$$reg);
  2916     emit_d32( cbuf, -1 );
  2917     // jcc(Assembler::below, exit);
  2918     emit_opcode( cbuf, 0x72 );
  2919     emit_d8    ( cbuf, 0x0C );
  2920     // movl(dst, equal_result);
  2921     emit_opcode( cbuf, 0xB8 + $dst$$reg);
  2922     emit_d32( cbuf, 0 );
  2923     // jcc(Assembler::equal, exit);
  2924     emit_opcode( cbuf, 0x74 );
  2925     emit_d8    ( cbuf, 0x05 );
  2926     // movl(dst, greater_result);
  2927     emit_opcode( cbuf, 0xB8 + $dst$$reg);
  2928     emit_d32( cbuf, 1 );
  2929   %}
  2932   // XMM version of CmpF_Result. Because the XMM compare
  2933   // instructions set the EFLAGS directly. It becomes simpler than
  2934   // the float version above.
  2935   enc_class CmpX_Result(eRegI dst) %{
  2936     MacroAssembler _masm(&cbuf);
  2937     Label nan, inc, done;
  2939     __ jccb(Assembler::parity, nan);
  2940     __ jccb(Assembler::equal,  done);
  2941     __ jccb(Assembler::above,  inc);
  2942     __ bind(nan);
  2943     __ decrement(as_Register($dst$$reg)); // NO L qqq
  2944     __ jmpb(done);
  2945     __ bind(inc);
  2946     __ increment(as_Register($dst$$reg)); // NO L qqq
  2947     __ bind(done);
  2948   %}
  2950   // Compare the longs and set flags
  2951   // BROKEN!  Do Not use as-is
  2952   enc_class cmpl_test( eRegL src1, eRegL src2 ) %{
  2953     // CMP    $src1.hi,$src2.hi
  2954     emit_opcode( cbuf, 0x3B );
  2955     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($src1$$reg), HIGH_FROM_LOW($src2$$reg) );
  2956     // JNE,s  done
  2957     emit_opcode(cbuf,0x75);
  2958     emit_d8(cbuf, 2 );
  2959     // CMP    $src1.lo,$src2.lo
  2960     emit_opcode( cbuf, 0x3B );
  2961     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
  2962 // done:
  2963   %}
  2965   enc_class convert_int_long( regL dst, eRegI src ) %{
  2966     // mov $dst.lo,$src
  2967     int dst_encoding = $dst$$reg;
  2968     int src_encoding = $src$$reg;
  2969     encode_Copy( cbuf, dst_encoding  , src_encoding );
  2970     // mov $dst.hi,$src
  2971     encode_Copy( cbuf, HIGH_FROM_LOW(dst_encoding), src_encoding );
  2972     // sar $dst.hi,31
  2973     emit_opcode( cbuf, 0xC1 );
  2974     emit_rm(cbuf, 0x3, 7, HIGH_FROM_LOW(dst_encoding) );
  2975     emit_d8(cbuf, 0x1F );
  2976   %}
  2978   enc_class convert_long_double( eRegL src ) %{
  2979     // push $src.hi
  2980     emit_opcode(cbuf, 0x50+HIGH_FROM_LOW($src$$reg));
  2981     // push $src.lo
  2982     emit_opcode(cbuf, 0x50+$src$$reg  );
  2983     // fild 64-bits at [SP]
  2984     emit_opcode(cbuf,0xdf);
  2985     emit_d8(cbuf, 0x6C);
  2986     emit_d8(cbuf, 0x24);
  2987     emit_d8(cbuf, 0x00);
  2988     // pop stack
  2989     emit_opcode(cbuf, 0x83); // add  SP, #8
  2990     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
  2991     emit_d8(cbuf, 0x8);
  2992   %}
  2994   enc_class multiply_con_and_shift_high( eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32_63 cnt, eFlagsReg cr ) %{
  2995     // IMUL   EDX:EAX,$src1
  2996     emit_opcode( cbuf, 0xF7 );
  2997     emit_rm( cbuf, 0x3, 0x5, $src1$$reg );
  2998     // SAR    EDX,$cnt-32
  2999     int shift_count = ((int)$cnt$$constant) - 32;
  3000     if (shift_count > 0) {
  3001       emit_opcode(cbuf, 0xC1);
  3002       emit_rm(cbuf, 0x3, 7, $dst$$reg );
  3003       emit_d8(cbuf, shift_count);
  3005   %}
  3007   // this version doesn't have add sp, 8
  3008   enc_class convert_long_double2( eRegL src ) %{
  3009     // push $src.hi
  3010     emit_opcode(cbuf, 0x50+HIGH_FROM_LOW($src$$reg));
  3011     // push $src.lo
  3012     emit_opcode(cbuf, 0x50+$src$$reg  );
  3013     // fild 64-bits at [SP]
  3014     emit_opcode(cbuf,0xdf);
  3015     emit_d8(cbuf, 0x6C);
  3016     emit_d8(cbuf, 0x24);
  3017     emit_d8(cbuf, 0x00);
  3018   %}
  3020   enc_class long_int_multiply( eADXRegL dst, nadxRegI src) %{
  3021     // Basic idea: long = (long)int * (long)int
  3022     // IMUL EDX:EAX, src
  3023     emit_opcode( cbuf, 0xF7 );
  3024     emit_rm( cbuf, 0x3, 0x5, $src$$reg);
  3025   %}
  3027   enc_class long_uint_multiply( eADXRegL dst, nadxRegI src) %{
  3028     // Basic Idea:  long = (int & 0xffffffffL) * (int & 0xffffffffL)
  3029     // MUL EDX:EAX, src
  3030     emit_opcode( cbuf, 0xF7 );
  3031     emit_rm( cbuf, 0x3, 0x4, $src$$reg);
  3032   %}
  3034   enc_class long_multiply( eADXRegL dst, eRegL src, eRegI tmp ) %{
  3035     // Basic idea: lo(result) = lo(x_lo * y_lo)
  3036     //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
  3037     // MOV    $tmp,$src.lo
  3038     encode_Copy( cbuf, $tmp$$reg, $src$$reg );
  3039     // IMUL   $tmp,EDX
  3040     emit_opcode( cbuf, 0x0F );
  3041     emit_opcode( cbuf, 0xAF );
  3042     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
  3043     // MOV    EDX,$src.hi
  3044     encode_Copy( cbuf, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg) );
  3045     // IMUL   EDX,EAX
  3046     emit_opcode( cbuf, 0x0F );
  3047     emit_opcode( cbuf, 0xAF );
  3048     emit_rm( cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg );
  3049     // ADD    $tmp,EDX
  3050     emit_opcode( cbuf, 0x03 );
  3051     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
  3052     // MUL   EDX:EAX,$src.lo
  3053     emit_opcode( cbuf, 0xF7 );
  3054     emit_rm( cbuf, 0x3, 0x4, $src$$reg );
  3055     // ADD    EDX,ESI
  3056     emit_opcode( cbuf, 0x03 );
  3057     emit_rm( cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $tmp$$reg );
  3058   %}
  3060   enc_class long_multiply_con( eADXRegL dst, immL_127 src, eRegI tmp ) %{
  3061     // Basic idea: lo(result) = lo(src * y_lo)
  3062     //             hi(result) = hi(src * y_lo) + lo(src * y_hi)
  3063     // IMUL   $tmp,EDX,$src
  3064     emit_opcode( cbuf, 0x6B );
  3065     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
  3066     emit_d8( cbuf, (int)$src$$constant );
  3067     // MOV    EDX,$src
  3068     emit_opcode(cbuf, 0xB8 + EDX_enc);
  3069     emit_d32( cbuf, (int)$src$$constant );
  3070     // MUL   EDX:EAX,EDX
  3071     emit_opcode( cbuf, 0xF7 );
  3072     emit_rm( cbuf, 0x3, 0x4, EDX_enc );
  3073     // ADD    EDX,ESI
  3074     emit_opcode( cbuf, 0x03 );
  3075     emit_rm( cbuf, 0x3, EDX_enc, $tmp$$reg );
  3076   %}
  3078   enc_class long_div( eRegL src1, eRegL src2 ) %{
  3079     // PUSH src1.hi
  3080     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src1$$reg) );
  3081     // PUSH src1.lo
  3082     emit_opcode(cbuf,               0x50+$src1$$reg  );
  3083     // PUSH src2.hi
  3084     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src2$$reg) );
  3085     // PUSH src2.lo
  3086     emit_opcode(cbuf,               0x50+$src2$$reg  );
  3087     // CALL directly to the runtime
  3088     cbuf.set_insts_mark();
  3089     emit_opcode(cbuf,0xE8);       // Call into runtime
  3090     emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::ldiv) - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
  3091     // Restore stack
  3092     emit_opcode(cbuf, 0x83); // add  SP, #framesize
  3093     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
  3094     emit_d8(cbuf, 4*4);
  3095   %}
  3097   enc_class long_mod( eRegL src1, eRegL src2 ) %{
  3098     // PUSH src1.hi
  3099     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src1$$reg) );
  3100     // PUSH src1.lo
  3101     emit_opcode(cbuf,               0x50+$src1$$reg  );
  3102     // PUSH src2.hi
  3103     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src2$$reg) );
  3104     // PUSH src2.lo
  3105     emit_opcode(cbuf,               0x50+$src2$$reg  );
  3106     // CALL directly to the runtime
  3107     cbuf.set_insts_mark();
  3108     emit_opcode(cbuf,0xE8);       // Call into runtime
  3109     emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::lrem ) - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
  3110     // Restore stack
  3111     emit_opcode(cbuf, 0x83); // add  SP, #framesize
  3112     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
  3113     emit_d8(cbuf, 4*4);
  3114   %}
  3116   enc_class long_cmp_flags0( eRegL src, eRegI tmp ) %{
  3117     // MOV   $tmp,$src.lo
  3118     emit_opcode(cbuf, 0x8B);
  3119     emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);
  3120     // OR    $tmp,$src.hi
  3121     emit_opcode(cbuf, 0x0B);
  3122     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg));
  3123   %}
  3125   enc_class long_cmp_flags1( eRegL src1, eRegL src2 ) %{
  3126     // CMP    $src1.lo,$src2.lo
  3127     emit_opcode( cbuf, 0x3B );
  3128     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
  3129     // JNE,s  skip
  3130     emit_cc(cbuf, 0x70, 0x5);
  3131     emit_d8(cbuf,2);
  3132     // CMP    $src1.hi,$src2.hi
  3133     emit_opcode( cbuf, 0x3B );
  3134     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($src1$$reg), HIGH_FROM_LOW($src2$$reg) );
  3135   %}
  3137   enc_class long_cmp_flags2( eRegL src1, eRegL src2, eRegI tmp ) %{
  3138     // CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits
  3139     emit_opcode( cbuf, 0x3B );
  3140     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
  3141     // MOV    $tmp,$src1.hi
  3142     emit_opcode( cbuf, 0x8B );
  3143     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src1$$reg) );
  3144     // SBB   $tmp,$src2.hi\t! Compute flags for long compare
  3145     emit_opcode( cbuf, 0x1B );
  3146     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src2$$reg) );
  3147   %}
  3149   enc_class long_cmp_flags3( eRegL src, eRegI tmp ) %{
  3150     // XOR    $tmp,$tmp
  3151     emit_opcode(cbuf,0x33);  // XOR
  3152     emit_rm(cbuf,0x3, $tmp$$reg, $tmp$$reg);
  3153     // CMP    $tmp,$src.lo
  3154     emit_opcode( cbuf, 0x3B );
  3155     emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg );
  3156     // SBB    $tmp,$src.hi
  3157     emit_opcode( cbuf, 0x1B );
  3158     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg) );
  3159   %}
  3161  // Sniff, sniff... smells like Gnu Superoptimizer
  3162   enc_class neg_long( eRegL dst ) %{
  3163     emit_opcode(cbuf,0xF7);    // NEG hi
  3164     emit_rm    (cbuf,0x3, 0x3, HIGH_FROM_LOW($dst$$reg));
  3165     emit_opcode(cbuf,0xF7);    // NEG lo
  3166     emit_rm    (cbuf,0x3, 0x3,               $dst$$reg );
  3167     emit_opcode(cbuf,0x83);    // SBB hi,0
  3168     emit_rm    (cbuf,0x3, 0x3, HIGH_FROM_LOW($dst$$reg));
  3169     emit_d8    (cbuf,0 );
  3170   %}
  3172   enc_class movq_ld(regXD dst, memory mem) %{
  3173     MacroAssembler _masm(&cbuf);
  3174     __ movq($dst$$XMMRegister, $mem$$Address);
  3175   %}
  3177   enc_class movq_st(memory mem, regXD src) %{
  3178     MacroAssembler _masm(&cbuf);
  3179     __ movq($mem$$Address, $src$$XMMRegister);
  3180   %}
  3182   enc_class pshufd_8x8(regX dst, regX src) %{
  3183     MacroAssembler _masm(&cbuf);
  3185     encode_CopyXD(cbuf, $dst$$reg, $src$$reg);
  3186     __ punpcklbw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg));
  3187     __ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg), 0x00);
  3188   %}
  3190   enc_class pshufd_4x16(regX dst, regX src) %{
  3191     MacroAssembler _masm(&cbuf);
  3193     __ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), 0x00);
  3194   %}
  3196   enc_class pshufd(regXD dst, regXD src, int mode) %{
  3197     MacroAssembler _masm(&cbuf);
  3199     __ pshufd(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), $mode);
  3200   %}
  3202   enc_class pxor(regXD dst, regXD src) %{
  3203     MacroAssembler _masm(&cbuf);
  3205     __ pxor(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg));
  3206   %}
  3208   enc_class mov_i2x(regXD dst, eRegI src) %{
  3209     MacroAssembler _masm(&cbuf);
  3211     __ movdl(as_XMMRegister($dst$$reg), as_Register($src$$reg));
  3212   %}
  3215   // Because the transitions from emitted code to the runtime
  3216   // monitorenter/exit helper stubs are so slow it's critical that
  3217   // we inline both the stack-locking fast-path and the inflated fast path.
  3218   //
  3219   // See also: cmpFastLock and cmpFastUnlock.
  3220   //
  3221   // What follows is a specialized inline transliteration of the code
  3222   // in slow_enter() and slow_exit().  If we're concerned about I$ bloat
  3223   // another option would be to emit TrySlowEnter and TrySlowExit methods
  3224   // at startup-time.  These methods would accept arguments as
  3225   // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
  3226   // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply
  3227   // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
  3228   // In practice, however, the # of lock sites is bounded and is usually small.
  3229   // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
  3230   // if the processor uses simple bimodal branch predictors keyed by EIP
  3231   // Since the helper routines would be called from multiple synchronization
  3232   // sites.
  3233   //
  3234   // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
  3235   // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
  3236   // to those specialized methods.  That'd give us a mostly platform-independent
  3237   // implementation that the JITs could optimize and inline at their pleasure.
  3238   // Done correctly, the only time we'd need to cross to native could would be
  3239   // to park() or unpark() threads.  We'd also need a few more unsafe operators
  3240   // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
  3241   // (b) explicit barriers or fence operations.
  3242   //
  3243   // TODO:
  3244   //
  3245   // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).
  3246   //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.
  3247   //    Given TLAB allocation, Self is usually manifested in a register, so passing it into
  3248   //    the lock operators would typically be faster than reifying Self.
  3249   //
  3250   // *  Ideally I'd define the primitives as:
  3251   //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
  3252   //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
  3253   //    Unfortunately ADLC bugs prevent us from expressing the ideal form.
  3254   //    Instead, we're stuck with a rather awkward and brittle register assignments below.
  3255   //    Furthermore the register assignments are overconstrained, possibly resulting in
  3256   //    sub-optimal code near the synchronization site.
  3257   //
  3258   // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.
  3259   //    Alternately, use a better sp-proximity test.
  3260   //
  3261   // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
  3262   //    Either one is sufficient to uniquely identify a thread.
  3263   //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
  3264   //
  3265   // *  Intrinsify notify() and notifyAll() for the common cases where the
  3266   //    object is locked by the calling thread but the waitlist is empty.
  3267   //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
  3268   //
  3269   // *  use jccb and jmpb instead of jcc and jmp to improve code density.
  3270   //    But beware of excessive branch density on AMD Opterons.
  3271   //
  3272   // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success
  3273   //    or failure of the fast-path.  If the fast-path fails then we pass
  3274   //    control to the slow-path, typically in C.  In Fast_Lock and
  3275   //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2
  3276   //    will emit a conditional branch immediately after the node.
  3277   //    So we have branches to branches and lots of ICC.ZF games.
  3278   //    Instead, it might be better to have C2 pass a "FailureLabel"
  3279   //    into Fast_Lock and Fast_Unlock.  In the case of success, control
  3280   //    will drop through the node.  ICC.ZF is undefined at exit.
  3281   //    In the case of failure, the node will branch directly to the
  3282   //    FailureLabel
  3285   // obj: object to lock
  3286   // box: on-stack box address (displaced header location) - KILLED
  3287   // rax,: tmp -- KILLED
  3288   // scr: tmp -- KILLED
  3289   enc_class Fast_Lock( eRegP obj, eRegP box, eAXRegI tmp, eRegP scr ) %{
  3291     Register objReg = as_Register($obj$$reg);
  3292     Register boxReg = as_Register($box$$reg);
  3293     Register tmpReg = as_Register($tmp$$reg);
  3294     Register scrReg = as_Register($scr$$reg);
  3296     // Ensure the register assignents are disjoint
  3297     guarantee (objReg != boxReg, "") ;
  3298     guarantee (objReg != tmpReg, "") ;
  3299     guarantee (objReg != scrReg, "") ;
  3300     guarantee (boxReg != tmpReg, "") ;
  3301     guarantee (boxReg != scrReg, "") ;
  3302     guarantee (tmpReg == as_Register(EAX_enc), "") ;
  3304     MacroAssembler masm(&cbuf);
  3306     if (_counters != NULL) {
  3307       masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
  3309     if (EmitSync & 1) {
  3310         // set box->dhw = unused_mark (3)
  3311         // Force all sync thru slow-path: slow_enter() and slow_exit() 
  3312         masm.movptr (Address(boxReg, 0), int32_t(markOopDesc::unused_mark())) ;             
  3313         masm.cmpptr (rsp, (int32_t)0) ;                        
  3314     } else 
  3315     if (EmitSync & 2) { 
  3316         Label DONE_LABEL ;           
  3317         if (UseBiasedLocking) {
  3318            // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
  3319            masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
  3322         masm.movptr(tmpReg, Address(objReg, 0)) ;          // fetch markword 
  3323         masm.orptr (tmpReg, 0x1);
  3324         masm.movptr(Address(boxReg, 0), tmpReg);           // Anticipate successful CAS 
  3325         if (os::is_MP()) { masm.lock();  }
  3326         masm.cmpxchgptr(boxReg, Address(objReg, 0));          // Updates tmpReg
  3327         masm.jcc(Assembler::equal, DONE_LABEL);
  3328         // Recursive locking
  3329         masm.subptr(tmpReg, rsp);
  3330         masm.andptr(tmpReg, (int32_t) 0xFFFFF003 );
  3331         masm.movptr(Address(boxReg, 0), tmpReg);
  3332         masm.bind(DONE_LABEL) ; 
  3333     } else {  
  3334       // Possible cases that we'll encounter in fast_lock 
  3335       // ------------------------------------------------
  3336       // * Inflated
  3337       //    -- unlocked
  3338       //    -- Locked
  3339       //       = by self
  3340       //       = by other
  3341       // * biased
  3342       //    -- by Self
  3343       //    -- by other
  3344       // * neutral
  3345       // * stack-locked
  3346       //    -- by self
  3347       //       = sp-proximity test hits
  3348       //       = sp-proximity test generates false-negative
  3349       //    -- by other
  3350       //
  3352       Label IsInflated, DONE_LABEL, PopDone ;
  3354       // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
  3355       // order to reduce the number of conditional branches in the most common cases.
  3356       // Beware -- there's a subtle invariant that fetch of the markword
  3357       // at [FETCH], below, will never observe a biased encoding (*101b).
  3358       // If this invariant is not held we risk exclusion (safety) failure.
  3359       if (UseBiasedLocking && !UseOptoBiasInlining) {
  3360         masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
  3363       masm.movptr(tmpReg, Address(objReg, 0)) ;         // [FETCH]
  3364       masm.testptr(tmpReg, 0x02) ;                      // Inflated v (Stack-locked or neutral)
  3365       masm.jccb  (Assembler::notZero, IsInflated) ;
  3367       // Attempt stack-locking ...
  3368       masm.orptr (tmpReg, 0x1);
  3369       masm.movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
  3370       if (os::is_MP()) { masm.lock();  }
  3371       masm.cmpxchgptr(boxReg, Address(objReg, 0));           // Updates tmpReg
  3372       if (_counters != NULL) {
  3373         masm.cond_inc32(Assembler::equal,
  3374                         ExternalAddress((address)_counters->fast_path_entry_count_addr()));
  3376       masm.jccb (Assembler::equal, DONE_LABEL);
  3378       // Recursive locking
  3379       masm.subptr(tmpReg, rsp);
  3380       masm.andptr(tmpReg, 0xFFFFF003 );
  3381       masm.movptr(Address(boxReg, 0), tmpReg);
  3382       if (_counters != NULL) {
  3383         masm.cond_inc32(Assembler::equal,
  3384                         ExternalAddress((address)_counters->fast_path_entry_count_addr()));
  3386       masm.jmp  (DONE_LABEL) ;
  3388       masm.bind (IsInflated) ;
  3390       // The object is inflated.
  3391       //
  3392       // TODO-FIXME: eliminate the ugly use of manifest constants:
  3393       //   Use markOopDesc::monitor_value instead of "2".
  3394       //   use markOop::unused_mark() instead of "3".
  3395       // The tmpReg value is an objectMonitor reference ORed with
  3396       // markOopDesc::monitor_value (2).   We can either convert tmpReg to an
  3397       // objectmonitor pointer by masking off the "2" bit or we can just
  3398       // use tmpReg as an objectmonitor pointer but bias the objectmonitor
  3399       // field offsets with "-2" to compensate for and annul the low-order tag bit.
  3400       //
  3401       // I use the latter as it avoids AGI stalls.
  3402       // As such, we write "mov r, [tmpReg+OFFSETOF(Owner)-2]"
  3403       // instead of "mov r, [tmpReg+OFFSETOF(Owner)]".
  3404       //
  3405       #define OFFSET_SKEWED(f) ((ObjectMonitor::f ## _offset_in_bytes())-2)
  3407       // boxReg refers to the on-stack BasicLock in the current frame.
  3408       // We'd like to write:
  3409       //   set box->_displaced_header = markOop::unused_mark().  Any non-0 value suffices.
  3410       // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers
  3411       // additional latency as we have another ST in the store buffer that must drain.
  3413       if (EmitSync & 8192) { 
  3414          masm.movptr(Address(boxReg, 0), 3) ;            // results in ST-before-CAS penalty
  3415          masm.get_thread (scrReg) ; 
  3416          masm.movptr(boxReg, tmpReg);                    // consider: LEA box, [tmp-2] 
  3417          masm.movptr(tmpReg, NULL_WORD);                 // consider: xor vs mov
  3418          if (os::is_MP()) { masm.lock(); } 
  3419          masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 
  3420       } else 
  3421       if ((EmitSync & 128) == 0) {                      // avoid ST-before-CAS
  3422          masm.movptr(scrReg, boxReg) ; 
  3423          masm.movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2] 
  3425          // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
  3426          if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
  3427             // prefetchw [eax + Offset(_owner)-2]
  3428             masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2));
  3431          if ((EmitSync & 64) == 0) {
  3432            // Optimistic form: consider XORL tmpReg,tmpReg
  3433            masm.movptr(tmpReg, NULL_WORD) ; 
  3434          } else { 
  3435            // Can suffer RTS->RTO upgrades on shared or cold $ lines
  3436            // Test-And-CAS instead of CAS
  3437            masm.movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;   // rax, = m->_owner
  3438            masm.testptr(tmpReg, tmpReg) ;                   // Locked ? 
  3439            masm.jccb  (Assembler::notZero, DONE_LABEL) ;                   
  3442          // Appears unlocked - try to swing _owner from null to non-null.
  3443          // Ideally, I'd manifest "Self" with get_thread and then attempt
  3444          // to CAS the register containing Self into m->Owner.
  3445          // But we don't have enough registers, so instead we can either try to CAS
  3446          // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
  3447          // we later store "Self" into m->Owner.  Transiently storing a stack address
  3448          // (rsp or the address of the box) into  m->owner is harmless.
  3449          // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
  3450          if (os::is_MP()) { masm.lock();  }
  3451          masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 
  3452          masm.movptr(Address(scrReg, 0), 3) ;          // box->_displaced_header = 3
  3453          masm.jccb  (Assembler::notZero, DONE_LABEL) ; 
  3454          masm.get_thread (scrReg) ;                    // beware: clobbers ICCs
  3455          masm.movptr(Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2), scrReg) ; 
  3456          masm.xorptr(boxReg, boxReg) ;                 // set icc.ZFlag = 1 to indicate success
  3458          // If the CAS fails we can either retry or pass control to the slow-path.  
  3459          // We use the latter tactic.  
  3460          // Pass the CAS result in the icc.ZFlag into DONE_LABEL
  3461          // If the CAS was successful ...
  3462          //   Self has acquired the lock
  3463          //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
  3464          // Intentional fall-through into DONE_LABEL ...
  3465       } else {
  3466          masm.movptr(Address(boxReg, 0), 3) ;       // results in ST-before-CAS penalty
  3467          masm.movptr(boxReg, tmpReg) ; 
  3469          // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
  3470          if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
  3471             // prefetchw [eax + Offset(_owner)-2]
  3472             masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2));
  3475          if ((EmitSync & 64) == 0) {
  3476            // Optimistic form
  3477            masm.xorptr  (tmpReg, tmpReg) ; 
  3478          } else { 
  3479            // Can suffer RTS->RTO upgrades on shared or cold $ lines
  3480            masm.movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;   // rax, = m->_owner
  3481            masm.testptr(tmpReg, tmpReg) ;                   // Locked ? 
  3482            masm.jccb  (Assembler::notZero, DONE_LABEL) ;                   
  3485          // Appears unlocked - try to swing _owner from null to non-null.
  3486          // Use either "Self" (in scr) or rsp as thread identity in _owner.
  3487          // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
  3488          masm.get_thread (scrReg) ;
  3489          if (os::is_MP()) { masm.lock(); }
  3490          masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
  3492          // If the CAS fails we can either retry or pass control to the slow-path.
  3493          // We use the latter tactic.
  3494          // Pass the CAS result in the icc.ZFlag into DONE_LABEL
  3495          // If the CAS was successful ...
  3496          //   Self has acquired the lock
  3497          //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
  3498          // Intentional fall-through into DONE_LABEL ...
  3501       // DONE_LABEL is a hot target - we'd really like to place it at the
  3502       // start of cache line by padding with NOPs.
  3503       // See the AMD and Intel software optimization manuals for the
  3504       // most efficient "long" NOP encodings.
  3505       // Unfortunately none of our alignment mechanisms suffice.
  3506       masm.bind(DONE_LABEL);
  3508       // Avoid branch-to-branch on AMD processors
  3509       // This appears to be superstition.
  3510       if (EmitSync & 32) masm.nop() ;
  3513       // At DONE_LABEL the icc ZFlag is set as follows ...
  3514       // Fast_Unlock uses the same protocol.
  3515       // ZFlag == 1 -> Success
  3516       // ZFlag == 0 -> Failure - force control through the slow-path
  3518   %}
  3520   // obj: object to unlock
  3521   // box: box address (displaced header location), killed.  Must be EAX.
  3522   // rbx,: killed tmp; cannot be obj nor box.
  3523   //
  3524   // Some commentary on balanced locking:
  3525   //
  3526   // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.
  3527   // Methods that don't have provably balanced locking are forced to run in the
  3528   // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
  3529   // The interpreter provides two properties:
  3530   // I1:  At return-time the interpreter automatically and quietly unlocks any
  3531   //      objects acquired the current activation (frame).  Recall that the
  3532   //      interpreter maintains an on-stack list of locks currently held by
  3533   //      a frame.
  3534   // I2:  If a method attempts to unlock an object that is not held by the
  3535   //      the frame the interpreter throws IMSX.
  3536   //
  3537   // Lets say A(), which has provably balanced locking, acquires O and then calls B().
  3538   // B() doesn't have provably balanced locking so it runs in the interpreter.
  3539   // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O
  3540   // is still locked by A().
  3541   //
  3542   // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
  3543   // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
  3544   // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
  3545   // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
  3547   enc_class Fast_Unlock( nabxRegP obj, eAXRegP box, eRegP tmp) %{
  3549     Register objReg = as_Register($obj$$reg);
  3550     Register boxReg = as_Register($box$$reg);
  3551     Register tmpReg = as_Register($tmp$$reg);
  3553     guarantee (objReg != boxReg, "") ;
  3554     guarantee (objReg != tmpReg, "") ;
  3555     guarantee (boxReg != tmpReg, "") ;
  3556     guarantee (boxReg == as_Register(EAX_enc), "") ;
  3557     MacroAssembler masm(&cbuf);
  3559     if (EmitSync & 4) {
  3560       // Disable - inhibit all inlining.  Force control through the slow-path
  3561       masm.cmpptr (rsp, 0) ; 
  3562     } else 
  3563     if (EmitSync & 8) {
  3564       Label DONE_LABEL ;
  3565       if (UseBiasedLocking) {
  3566          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
  3568       // classic stack-locking code ...
  3569       masm.movptr(tmpReg, Address(boxReg, 0)) ;
  3570       masm.testptr(tmpReg, tmpReg) ;
  3571       masm.jcc   (Assembler::zero, DONE_LABEL) ;
  3572       if (os::is_MP()) { masm.lock(); }
  3573       masm.cmpxchgptr(tmpReg, Address(objReg, 0));          // Uses EAX which is box
  3574       masm.bind(DONE_LABEL);
  3575     } else {
  3576       Label DONE_LABEL, Stacked, CheckSucc, Inflated ;
  3578       // Critically, the biased locking test must have precedence over
  3579       // and appear before the (box->dhw == 0) recursive stack-lock test.
  3580       if (UseBiasedLocking && !UseOptoBiasInlining) {
  3581          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
  3584       masm.cmpptr(Address(boxReg, 0), 0) ;            // Examine the displaced header
  3585       masm.movptr(tmpReg, Address(objReg, 0)) ;       // Examine the object's markword
  3586       masm.jccb  (Assembler::zero, DONE_LABEL) ;      // 0 indicates recursive stack-lock
  3588       masm.testptr(tmpReg, 0x02) ;                     // Inflated? 
  3589       masm.jccb  (Assembler::zero, Stacked) ;
  3591       masm.bind  (Inflated) ;
  3592       // It's inflated.
  3593       // Despite our balanced locking property we still check that m->_owner == Self
  3594       // as java routines or native JNI code called by this thread might
  3595       // have released the lock.
  3596       // Refer to the comments in synchronizer.cpp for how we might encode extra
  3597       // state in _succ so we can avoid fetching EntryList|cxq.
  3598       //
  3599       // I'd like to add more cases in fast_lock() and fast_unlock() --
  3600       // such as recursive enter and exit -- but we have to be wary of
  3601       // I$ bloat, T$ effects and BP$ effects.
  3602       //
  3603       // If there's no contention try a 1-0 exit.  That is, exit without
  3604       // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
  3605       // we detect and recover from the race that the 1-0 exit admits.
  3606       //
  3607       // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier
  3608       // before it STs null into _owner, releasing the lock.  Updates
  3609       // to data protected by the critical section must be visible before
  3610       // we drop the lock (and thus before any other thread could acquire
  3611       // the lock and observe the fields protected by the lock).
  3612       // IA32's memory-model is SPO, so STs are ordered with respect to
  3613       // each other and there's no need for an explicit barrier (fence).
  3614       // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
  3616       masm.get_thread (boxReg) ;
  3617       if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
  3618         // prefetchw [ebx + Offset(_owner)-2]
  3619         masm.prefetchw(Address(rbx, ObjectMonitor::owner_offset_in_bytes()-2));
  3622       // Note that we could employ various encoding schemes to reduce
  3623       // the number of loads below (currently 4) to just 2 or 3.
  3624       // Refer to the comments in synchronizer.cpp.
  3625       // In practice the chain of fetches doesn't seem to impact performance, however.
  3626       if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {
  3627          // Attempt to reduce branch density - AMD's branch predictor.
  3628          masm.xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;  
  3629          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
  3630          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; 
  3631          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; 
  3632          masm.jccb  (Assembler::notZero, DONE_LABEL) ; 
  3633          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD) ; 
  3634          masm.jmpb  (DONE_LABEL) ; 
  3635       } else { 
  3636          masm.xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;  
  3637          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
  3638          masm.jccb  (Assembler::notZero, DONE_LABEL) ; 
  3639          masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; 
  3640          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; 
  3641          masm.jccb  (Assembler::notZero, CheckSucc) ; 
  3642          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD) ; 
  3643          masm.jmpb  (DONE_LABEL) ; 
  3646       // The Following code fragment (EmitSync & 65536) improves the performance of
  3647       // contended applications and contended synchronization microbenchmarks.
  3648       // Unfortunately the emission of the code - even though not executed - causes regressions
  3649       // in scimark and jetstream, evidently because of $ effects.  Replacing the code
  3650       // with an equal number of never-executed NOPs results in the same regression.
  3651       // We leave it off by default.
  3653       if ((EmitSync & 65536) != 0) {
  3654          Label LSuccess, LGoSlowPath ;
  3656          masm.bind  (CheckSucc) ;
  3658          // Optional pre-test ... it's safe to elide this
  3659          if ((EmitSync & 16) == 0) { 
  3660             masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; 
  3661             masm.jccb  (Assembler::zero, LGoSlowPath) ; 
  3664          // We have a classic Dekker-style idiom:
  3665          //    ST m->_owner = 0 ; MEMBAR; LD m->_succ
  3666          // There are a number of ways to implement the barrier:
  3667          // (1) lock:andl &m->_owner, 0
  3668          //     is fast, but mask doesn't currently support the "ANDL M,IMM32" form.
  3669          //     LOCK: ANDL [ebx+Offset(_Owner)-2], 0
  3670          //     Encodes as 81 31 OFF32 IMM32 or 83 63 OFF8 IMM8
  3671          // (2) If supported, an explicit MFENCE is appealing.
  3672          //     In older IA32 processors MFENCE is slower than lock:add or xchg
  3673          //     particularly if the write-buffer is full as might be the case if
  3674          //     if stores closely precede the fence or fence-equivalent instruction.
  3675          //     In more modern implementations MFENCE appears faster, however.
  3676          // (3) In lieu of an explicit fence, use lock:addl to the top-of-stack
  3677          //     The $lines underlying the top-of-stack should be in M-state.
  3678          //     The locked add instruction is serializing, of course.
  3679          // (4) Use xchg, which is serializing
  3680          //     mov boxReg, 0; xchgl boxReg, [tmpReg + Offset(_owner)-2] also works
  3681          // (5) ST m->_owner = 0 and then execute lock:orl &m->_succ, 0.
  3682          //     The integer condition codes will tell us if succ was 0.
  3683          //     Since _succ and _owner should reside in the same $line and
  3684          //     we just stored into _owner, it's likely that the $line
  3685          //     remains in M-state for the lock:orl.
  3686          //
  3687          // We currently use (3), although it's likely that switching to (2)
  3688          // is correct for the future.
  3690          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD) ; 
  3691          if (os::is_MP()) { 
  3692             if (VM_Version::supports_sse2() && 1 == FenceInstruction) { 
  3693               masm.mfence();
  3694             } else { 
  3695               masm.lock () ; masm.addptr(Address(rsp, 0), 0) ; 
  3698          // Ratify _succ remains non-null
  3699          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; 
  3700          masm.jccb  (Assembler::notZero, LSuccess) ; 
  3702          masm.xorptr(boxReg, boxReg) ;                  // box is really EAX
  3703          if (os::is_MP()) { masm.lock(); }
  3704          masm.cmpxchgptr(rsp, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
  3705          masm.jccb  (Assembler::notEqual, LSuccess) ;
  3706          // Since we're low on registers we installed rsp as a placeholding in _owner.
  3707          // Now install Self over rsp.  This is safe as we're transitioning from
  3708          // non-null to non=null
  3709          masm.get_thread (boxReg) ;
  3710          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), boxReg) ;
  3711          // Intentional fall-through into LGoSlowPath ...
  3713          masm.bind  (LGoSlowPath) ; 
  3714          masm.orptr(boxReg, 1) ;                      // set ICC.ZF=0 to indicate failure
  3715          masm.jmpb  (DONE_LABEL) ; 
  3717          masm.bind  (LSuccess) ; 
  3718          masm.xorptr(boxReg, boxReg) ;                 // set ICC.ZF=1 to indicate success
  3719          masm.jmpb  (DONE_LABEL) ; 
  3722       masm.bind (Stacked) ;
  3723       // It's not inflated and it's not recursively stack-locked and it's not biased.
  3724       // It must be stack-locked.
  3725       // Try to reset the header to displaced header.
  3726       // The "box" value on the stack is stable, so we can reload
  3727       // and be assured we observe the same value as above.
  3728       masm.movptr(tmpReg, Address(boxReg, 0)) ;
  3729       if (os::is_MP()) {   masm.lock();    }
  3730       masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses EAX which is box
  3731       // Intention fall-thru into DONE_LABEL
  3734       // DONE_LABEL is a hot target - we'd really like to place it at the
  3735       // start of cache line by padding with NOPs.
  3736       // See the AMD and Intel software optimization manuals for the
  3737       // most efficient "long" NOP encodings.
  3738       // Unfortunately none of our alignment mechanisms suffice.
  3739       if ((EmitSync & 65536) == 0) {
  3740          masm.bind (CheckSucc) ;
  3742       masm.bind(DONE_LABEL);
  3744       // Avoid branch to branch on AMD processors
  3745       if (EmitSync & 32768) { masm.nop() ; }
  3747   %}
  3750   enc_class enc_pop_rdx() %{
  3751     emit_opcode(cbuf,0x5A);
  3752   %}
  3754   enc_class enc_rethrow() %{
  3755     cbuf.set_insts_mark();
  3756     emit_opcode(cbuf, 0xE9);        // jmp    entry
  3757     emit_d32_reloc(cbuf, (int)OptoRuntime::rethrow_stub() - ((int)cbuf.insts_end())-4,
  3758                    runtime_call_Relocation::spec(), RELOC_IMM32 );
  3759   %}
  3762   // Convert a double to an int.  Java semantics require we do complex
  3763   // manglelations in the corner cases.  So we set the rounding mode to
  3764   // 'zero', store the darned double down as an int, and reset the
  3765   // rounding mode to 'nearest'.  The hardware throws an exception which
  3766   // patches up the correct value directly to the stack.
  3767   enc_class D2I_encoding( regD src ) %{
  3768     // Flip to round-to-zero mode.  We attempted to allow invalid-op
  3769     // exceptions here, so that a NAN or other corner-case value will
  3770     // thrown an exception (but normal values get converted at full speed).
  3771     // However, I2C adapters and other float-stack manglers leave pending
  3772     // invalid-op exceptions hanging.  We would have to clear them before
  3773     // enabling them and that is more expensive than just testing for the
  3774     // invalid value Intel stores down in the corner cases.
  3775     emit_opcode(cbuf,0xD9);            // FLDCW  trunc
  3776     emit_opcode(cbuf,0x2D);
  3777     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
  3778     // Allocate a word
  3779     emit_opcode(cbuf,0x83);            // SUB ESP,4
  3780     emit_opcode(cbuf,0xEC);
  3781     emit_d8(cbuf,0x04);
  3782     // Encoding assumes a double has been pushed into FPR0.
  3783     // Store down the double as an int, popping the FPU stack
  3784     emit_opcode(cbuf,0xDB);            // FISTP [ESP]
  3785     emit_opcode(cbuf,0x1C);
  3786     emit_d8(cbuf,0x24);
  3787     // Restore the rounding mode; mask the exception
  3788     emit_opcode(cbuf,0xD9);            // FLDCW   std/24-bit mode
  3789     emit_opcode(cbuf,0x2D);
  3790     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
  3791         ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
  3792         : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
  3794     // Load the converted int; adjust CPU stack
  3795     emit_opcode(cbuf,0x58);       // POP EAX
  3796     emit_opcode(cbuf,0x3D);       // CMP EAX,imm
  3797     emit_d32   (cbuf,0x80000000); //         0x80000000
  3798     emit_opcode(cbuf,0x75);       // JNE around_slow_call
  3799     emit_d8    (cbuf,0x07);       // Size of slow_call
  3800     // Push src onto stack slow-path
  3801     emit_opcode(cbuf,0xD9 );      // FLD     ST(i)
  3802     emit_d8    (cbuf,0xC0-1+$src$$reg );
  3803     // CALL directly to the runtime
  3804     cbuf.set_insts_mark();
  3805     emit_opcode(cbuf,0xE8);       // Call into runtime
  3806     emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
  3807     // Carry on here...
  3808   %}
  3810   enc_class D2L_encoding( regD src ) %{
  3811     emit_opcode(cbuf,0xD9);            // FLDCW  trunc
  3812     emit_opcode(cbuf,0x2D);
  3813     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
  3814     // Allocate a word
  3815     emit_opcode(cbuf,0x83);            // SUB ESP,8
  3816     emit_opcode(cbuf,0xEC);
  3817     emit_d8(cbuf,0x08);
  3818     // Encoding assumes a double has been pushed into FPR0.
  3819     // Store down the double as a long, popping the FPU stack
  3820     emit_opcode(cbuf,0xDF);            // FISTP [ESP]
  3821     emit_opcode(cbuf,0x3C);
  3822     emit_d8(cbuf,0x24);
  3823     // Restore the rounding mode; mask the exception
  3824     emit_opcode(cbuf,0xD9);            // FLDCW   std/24-bit mode
  3825     emit_opcode(cbuf,0x2D);
  3826     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
  3827         ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
  3828         : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
  3830     // Load the converted int; adjust CPU stack
  3831     emit_opcode(cbuf,0x58);       // POP EAX
  3832     emit_opcode(cbuf,0x5A);       // POP EDX
  3833     emit_opcode(cbuf,0x81);       // CMP EDX,imm
  3834     emit_d8    (cbuf,0xFA);       // rdx
  3835     emit_d32   (cbuf,0x80000000); //         0x80000000
  3836     emit_opcode(cbuf,0x75);       // JNE around_slow_call
  3837     emit_d8    (cbuf,0x07+4);     // Size of slow_call
  3838     emit_opcode(cbuf,0x85);       // TEST EAX,EAX
  3839     emit_opcode(cbuf,0xC0);       // 2/rax,/rax,
  3840     emit_opcode(cbuf,0x75);       // JNE around_slow_call
  3841     emit_d8    (cbuf,0x07);       // Size of slow_call
  3842     // Push src onto stack slow-path
  3843     emit_opcode(cbuf,0xD9 );      // FLD     ST(i)
  3844     emit_d8    (cbuf,0xC0-1+$src$$reg );
  3845     // CALL directly to the runtime
  3846     cbuf.set_insts_mark();
  3847     emit_opcode(cbuf,0xE8);       // Call into runtime
  3848     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
  3849     // Carry on here...
  3850   %}
  3852   enc_class X2L_encoding( regX src ) %{
  3853     // Allocate a word
  3854     emit_opcode(cbuf,0x83);      // SUB ESP,8
  3855     emit_opcode(cbuf,0xEC);
  3856     emit_d8(cbuf,0x08);
  3858     emit_opcode  (cbuf, 0xF3 );  // MOVSS [ESP], src
  3859     emit_opcode  (cbuf, 0x0F );
  3860     emit_opcode  (cbuf, 0x11 );
  3861     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
  3863     emit_opcode(cbuf,0xD9 );     // FLD_S [ESP]
  3864     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  3866     emit_opcode(cbuf,0xD9);      // FLDCW  trunc
  3867     emit_opcode(cbuf,0x2D);
  3868     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
  3870     // Encoding assumes a double has been pushed into FPR0.
  3871     // Store down the double as a long, popping the FPU stack
  3872     emit_opcode(cbuf,0xDF);      // FISTP [ESP]
  3873     emit_opcode(cbuf,0x3C);
  3874     emit_d8(cbuf,0x24);
  3876     // Restore the rounding mode; mask the exception
  3877     emit_opcode(cbuf,0xD9);      // FLDCW   std/24-bit mode
  3878     emit_opcode(cbuf,0x2D);
  3879     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
  3880       ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
  3881       : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
  3883     // Load the converted int; adjust CPU stack
  3884     emit_opcode(cbuf,0x58);      // POP EAX
  3886     emit_opcode(cbuf,0x5A);      // POP EDX
  3888     emit_opcode(cbuf,0x81);      // CMP EDX,imm
  3889     emit_d8    (cbuf,0xFA);      // rdx
  3890     emit_d32   (cbuf,0x80000000);//         0x80000000
  3892     emit_opcode(cbuf,0x75);      // JNE around_slow_call
  3893     emit_d8    (cbuf,0x13+4);    // Size of slow_call
  3895     emit_opcode(cbuf,0x85);      // TEST EAX,EAX
  3896     emit_opcode(cbuf,0xC0);      // 2/rax,/rax,
  3898     emit_opcode(cbuf,0x75);      // JNE around_slow_call
  3899     emit_d8    (cbuf,0x13);      // Size of slow_call
  3901     // Allocate a word
  3902     emit_opcode(cbuf,0x83);      // SUB ESP,4
  3903     emit_opcode(cbuf,0xEC);
  3904     emit_d8(cbuf,0x04);
  3906     emit_opcode  (cbuf, 0xF3 );  // MOVSS [ESP], src
  3907     emit_opcode  (cbuf, 0x0F );
  3908     emit_opcode  (cbuf, 0x11 );
  3909     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
  3911     emit_opcode(cbuf,0xD9 );     // FLD_S [ESP]
  3912     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  3914     emit_opcode(cbuf,0x83);      // ADD ESP,4
  3915     emit_opcode(cbuf,0xC4);
  3916     emit_d8(cbuf,0x04);
  3918     // CALL directly to the runtime
  3919     cbuf.set_insts_mark();
  3920     emit_opcode(cbuf,0xE8);       // Call into runtime
  3921     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
  3922     // Carry on here...
  3923   %}
  3925   enc_class XD2L_encoding( regXD src ) %{
  3926     // Allocate a word
  3927     emit_opcode(cbuf,0x83);      // SUB ESP,8
  3928     emit_opcode(cbuf,0xEC);
  3929     emit_d8(cbuf,0x08);
  3931     emit_opcode  (cbuf, 0xF2 );  // MOVSD [ESP], src
  3932     emit_opcode  (cbuf, 0x0F );
  3933     emit_opcode  (cbuf, 0x11 );
  3934     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
  3936     emit_opcode(cbuf,0xDD );     // FLD_D [ESP]
  3937     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  3939     emit_opcode(cbuf,0xD9);      // FLDCW  trunc
  3940     emit_opcode(cbuf,0x2D);
  3941     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
  3943     // Encoding assumes a double has been pushed into FPR0.
  3944     // Store down the double as a long, popping the FPU stack
  3945     emit_opcode(cbuf,0xDF);      // FISTP [ESP]
  3946     emit_opcode(cbuf,0x3C);
  3947     emit_d8(cbuf,0x24);
  3949     // Restore the rounding mode; mask the exception
  3950     emit_opcode(cbuf,0xD9);      // FLDCW   std/24-bit mode
  3951     emit_opcode(cbuf,0x2D);
  3952     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
  3953       ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
  3954       : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
  3956     // Load the converted int; adjust CPU stack
  3957     emit_opcode(cbuf,0x58);      // POP EAX
  3959     emit_opcode(cbuf,0x5A);      // POP EDX
  3961     emit_opcode(cbuf,0x81);      // CMP EDX,imm
  3962     emit_d8    (cbuf,0xFA);      // rdx
  3963     emit_d32   (cbuf,0x80000000); //         0x80000000
  3965     emit_opcode(cbuf,0x75);      // JNE around_slow_call
  3966     emit_d8    (cbuf,0x13+4);    // Size of slow_call
  3968     emit_opcode(cbuf,0x85);      // TEST EAX,EAX
  3969     emit_opcode(cbuf,0xC0);      // 2/rax,/rax,
  3971     emit_opcode(cbuf,0x75);      // JNE around_slow_call
  3972     emit_d8    (cbuf,0x13);      // Size of slow_call
  3974     // Push src onto stack slow-path
  3975     // Allocate a word
  3976     emit_opcode(cbuf,0x83);      // SUB ESP,8
  3977     emit_opcode(cbuf,0xEC);
  3978     emit_d8(cbuf,0x08);
  3980     emit_opcode  (cbuf, 0xF2 );  // MOVSD [ESP], src
  3981     emit_opcode  (cbuf, 0x0F );
  3982     emit_opcode  (cbuf, 0x11 );
  3983     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
  3985     emit_opcode(cbuf,0xDD );     // FLD_D [ESP]
  3986     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  3988     emit_opcode(cbuf,0x83);      // ADD ESP,8
  3989     emit_opcode(cbuf,0xC4);
  3990     emit_d8(cbuf,0x08);
  3992     // CALL directly to the runtime
  3993     cbuf.set_insts_mark();
  3994     emit_opcode(cbuf,0xE8);      // Call into runtime
  3995     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
  3996     // Carry on here...
  3997   %}
  3999   enc_class D2X_encoding( regX dst, regD src ) %{
  4000     // Allocate a word
  4001     emit_opcode(cbuf,0x83);            // SUB ESP,4
  4002     emit_opcode(cbuf,0xEC);
  4003     emit_d8(cbuf,0x04);
  4004     int pop = 0x02;
  4005     if ($src$$reg != FPR1L_enc) {
  4006       emit_opcode( cbuf, 0xD9 );       // FLD    ST(i-1)
  4007       emit_d8( cbuf, 0xC0-1+$src$$reg );
  4008       pop = 0x03;
  4010     store_to_stackslot( cbuf, 0xD9, pop, 0 ); // FST<P>_S  [ESP]
  4012     emit_opcode  (cbuf, 0xF3 );        // MOVSS dst(xmm), [ESP]
  4013     emit_opcode  (cbuf, 0x0F );
  4014     emit_opcode  (cbuf, 0x10 );
  4015     encode_RegMem(cbuf, $dst$$reg, ESP_enc, 0x4, 0, 0, false);
  4017     emit_opcode(cbuf,0x83);            // ADD ESP,4
  4018     emit_opcode(cbuf,0xC4);
  4019     emit_d8(cbuf,0x04);
  4020     // Carry on here...
  4021   %}
  4023   enc_class FX2I_encoding( regX src, eRegI dst ) %{
  4024     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
  4026     // Compare the result to see if we need to go to the slow path
  4027     emit_opcode(cbuf,0x81);       // CMP dst,imm
  4028     emit_rm    (cbuf,0x3,0x7,$dst$$reg);
  4029     emit_d32   (cbuf,0x80000000); //         0x80000000
  4031     emit_opcode(cbuf,0x75);       // JNE around_slow_call
  4032     emit_d8    (cbuf,0x13);       // Size of slow_call
  4033     // Store xmm to a temp memory
  4034     // location and push it onto stack.
  4036     emit_opcode(cbuf,0x83);  // SUB ESP,4
  4037     emit_opcode(cbuf,0xEC);
  4038     emit_d8(cbuf, $primary ? 0x8 : 0x4);
  4040     emit_opcode  (cbuf, $primary ? 0xF2 : 0xF3 );   // MOVSS [ESP], xmm
  4041     emit_opcode  (cbuf, 0x0F );
  4042     emit_opcode  (cbuf, 0x11 );
  4043     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
  4045     emit_opcode(cbuf, $primary ? 0xDD : 0xD9 );      // FLD [ESP]
  4046     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  4048     emit_opcode(cbuf,0x83);    // ADD ESP,4
  4049     emit_opcode(cbuf,0xC4);
  4050     emit_d8(cbuf, $primary ? 0x8 : 0x4);
  4052     // CALL directly to the runtime
  4053     cbuf.set_insts_mark();
  4054     emit_opcode(cbuf,0xE8);       // Call into runtime
  4055     emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
  4057     // Carry on here...
  4058   %}
  4060   enc_class X2D_encoding( regD dst, regX src ) %{
  4061     // Allocate a word
  4062     emit_opcode(cbuf,0x83);     // SUB ESP,4
  4063     emit_opcode(cbuf,0xEC);
  4064     emit_d8(cbuf,0x04);
  4066     emit_opcode  (cbuf, 0xF3 ); // MOVSS [ESP], xmm
  4067     emit_opcode  (cbuf, 0x0F );
  4068     emit_opcode  (cbuf, 0x11 );
  4069     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
  4071     emit_opcode(cbuf,0xD9 );    // FLD_S [ESP]
  4072     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
  4074     emit_opcode(cbuf,0x83);     // ADD ESP,4
  4075     emit_opcode(cbuf,0xC4);
  4076     emit_d8(cbuf,0x04);
  4078     // Carry on here...
  4079   %}
  4081   enc_class AbsXF_encoding(regX dst) %{
  4082     address signmask_address=(address)float_signmask_pool;
  4083     // andpd:\tANDPS  $dst,[signconst]
  4084     emit_opcode(cbuf, 0x0F);
  4085     emit_opcode(cbuf, 0x54);
  4086     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
  4087     emit_d32(cbuf, (int)signmask_address);
  4088   %}
  4090   enc_class AbsXD_encoding(regXD dst) %{
  4091     address signmask_address=(address)double_signmask_pool;
  4092     // andpd:\tANDPD  $dst,[signconst]
  4093     emit_opcode(cbuf, 0x66);
  4094     emit_opcode(cbuf, 0x0F);
  4095     emit_opcode(cbuf, 0x54);
  4096     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
  4097     emit_d32(cbuf, (int)signmask_address);
  4098   %}
  4100   enc_class NegXF_encoding(regX dst) %{
  4101     address signmask_address=(address)float_signflip_pool;
  4102     // andpd:\tXORPS  $dst,[signconst]
  4103     emit_opcode(cbuf, 0x0F);
  4104     emit_opcode(cbuf, 0x57);
  4105     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
  4106     emit_d32(cbuf, (int)signmask_address);
  4107   %}
  4109   enc_class NegXD_encoding(regXD dst) %{
  4110     address signmask_address=(address)double_signflip_pool;
  4111     // andpd:\tXORPD  $dst,[signconst]
  4112     emit_opcode(cbuf, 0x66);
  4113     emit_opcode(cbuf, 0x0F);
  4114     emit_opcode(cbuf, 0x57);
  4115     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
  4116     emit_d32(cbuf, (int)signmask_address);
  4117   %}
  4119   enc_class FMul_ST_reg( eRegF src1 ) %{
  4120     // Operand was loaded from memory into fp ST (stack top)
  4121     // FMUL   ST,$src  /* D8 C8+i */
  4122     emit_opcode(cbuf, 0xD8);
  4123     emit_opcode(cbuf, 0xC8 + $src1$$reg);
  4124   %}
  4126   enc_class FAdd_ST_reg( eRegF src2 ) %{
  4127     // FADDP  ST,src2  /* D8 C0+i */
  4128     emit_opcode(cbuf, 0xD8);
  4129     emit_opcode(cbuf, 0xC0 + $src2$$reg);
  4130     //could use FADDP  src2,fpST  /* DE C0+i */
  4131   %}
  4133   enc_class FAddP_reg_ST( eRegF src2 ) %{
  4134     // FADDP  src2,ST  /* DE C0+i */
  4135     emit_opcode(cbuf, 0xDE);
  4136     emit_opcode(cbuf, 0xC0 + $src2$$reg);
  4137   %}
  4139   enc_class subF_divF_encode( eRegF src1, eRegF src2) %{
  4140     // Operand has been loaded into fp ST (stack top)
  4141       // FSUB   ST,$src1
  4142       emit_opcode(cbuf, 0xD8);
  4143       emit_opcode(cbuf, 0xE0 + $src1$$reg);
  4145       // FDIV
  4146       emit_opcode(cbuf, 0xD8);
  4147       emit_opcode(cbuf, 0xF0 + $src2$$reg);
  4148   %}
  4150   enc_class MulFAddF (eRegF src1, eRegF src2) %{
  4151     // Operand was loaded from memory into fp ST (stack top)
  4152     // FADD   ST,$src  /* D8 C0+i */
  4153     emit_opcode(cbuf, 0xD8);
  4154     emit_opcode(cbuf, 0xC0 + $src1$$reg);
  4156     // FMUL  ST,src2  /* D8 C*+i */
  4157     emit_opcode(cbuf, 0xD8);
  4158     emit_opcode(cbuf, 0xC8 + $src2$$reg);
  4159   %}
  4162   enc_class MulFAddFreverse (eRegF src1, eRegF src2) %{
  4163     // Operand was loaded from memory into fp ST (stack top)
  4164     // FADD   ST,$src  /* D8 C0+i */
  4165     emit_opcode(cbuf, 0xD8);
  4166     emit_opcode(cbuf, 0xC0 + $src1$$reg);
  4168     // FMULP  src2,ST  /* DE C8+i */
  4169     emit_opcode(cbuf, 0xDE);
  4170     emit_opcode(cbuf, 0xC8 + $src2$$reg);
  4171   %}
  4173   // Atomically load the volatile long
  4174   enc_class enc_loadL_volatile( memory mem, stackSlotL dst ) %{
  4175     emit_opcode(cbuf,0xDF);
  4176     int rm_byte_opcode = 0x05;
  4177     int base     = $mem$$base;
  4178     int index    = $mem$$index;
  4179     int scale    = $mem$$scale;
  4180     int displace = $mem$$disp;
  4181     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
  4182     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_is_oop);
  4183     store_to_stackslot( cbuf, 0x0DF, 0x07, $dst$$disp );
  4184   %}
  4186   enc_class enc_loadLX_volatile( memory mem, stackSlotL dst, regXD tmp ) %{
  4187     { // Atomic long load
  4188       // UseXmmLoadAndClearUpper ? movsd $tmp,$mem : movlpd $tmp,$mem
  4189       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
  4190       emit_opcode(cbuf,0x0F);
  4191       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0x10 : 0x12);
  4192       int base     = $mem$$base;
  4193       int index    = $mem$$index;
  4194       int scale    = $mem$$scale;
  4195       int displace = $mem$$disp;
  4196       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
  4197       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
  4199     { // MOVSD $dst,$tmp ! atomic long store
  4200       emit_opcode(cbuf,0xF2);
  4201       emit_opcode(cbuf,0x0F);
  4202       emit_opcode(cbuf,0x11);
  4203       int base     = $dst$$base;
  4204       int index    = $dst$$index;
  4205       int scale    = $dst$$scale;
  4206       int displace = $dst$$disp;
  4207       bool disp_is_oop = $dst->disp_is_oop(); // disp-as-oop when working with static globals
  4208       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
  4210   %}
  4212   enc_class enc_loadLX_reg_volatile( memory mem, eRegL dst, regXD tmp ) %{
  4213     { // Atomic long load
  4214       // UseXmmLoadAndClearUpper ? movsd $tmp,$mem : movlpd $tmp,$mem
  4215       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
  4216       emit_opcode(cbuf,0x0F);
  4217       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0x10 : 0x12);
  4218       int base     = $mem$$base;
  4219       int index    = $mem$$index;
  4220       int scale    = $mem$$scale;
  4221       int displace = $mem$$disp;
  4222       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
  4223       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
  4225     { // MOVD $dst.lo,$tmp
  4226       emit_opcode(cbuf,0x66);
  4227       emit_opcode(cbuf,0x0F);
  4228       emit_opcode(cbuf,0x7E);
  4229       emit_rm(cbuf, 0x3, $tmp$$reg, $dst$$reg);
  4231     { // PSRLQ $tmp,32
  4232       emit_opcode(cbuf,0x66);
  4233       emit_opcode(cbuf,0x0F);
  4234       emit_opcode(cbuf,0x73);
  4235       emit_rm(cbuf, 0x3, 0x02, $tmp$$reg);
  4236       emit_d8(cbuf, 0x20);
  4238     { // MOVD $dst.hi,$tmp
  4239       emit_opcode(cbuf,0x66);
  4240       emit_opcode(cbuf,0x0F);
  4241       emit_opcode(cbuf,0x7E);
  4242       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg));
  4244   %}
  4246   // Volatile Store Long.  Must be atomic, so move it into
  4247   // the FP TOS and then do a 64-bit FIST.  Has to probe the
  4248   // target address before the store (for null-ptr checks)
  4249   // so the memory operand is used twice in the encoding.
  4250   enc_class enc_storeL_volatile( memory mem, stackSlotL src ) %{
  4251     store_to_stackslot( cbuf, 0x0DF, 0x05, $src$$disp );
  4252     cbuf.set_insts_mark();            // Mark start of FIST in case $mem has an oop
  4253     emit_opcode(cbuf,0xDF);
  4254     int rm_byte_opcode = 0x07;
  4255     int base     = $mem$$base;
  4256     int index    = $mem$$index;
  4257     int scale    = $mem$$scale;
  4258     int displace = $mem$$disp;
  4259     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
  4260     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_is_oop);
  4261   %}
  4263   enc_class enc_storeLX_volatile( memory mem, stackSlotL src, regXD tmp) %{
  4264     { // Atomic long load
  4265       // UseXmmLoadAndClearUpper ? movsd $tmp,[$src] : movlpd $tmp,[$src]
  4266       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
  4267       emit_opcode(cbuf,0x0F);
  4268       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0x10 : 0x12);
  4269       int base     = $src$$base;
  4270       int index    = $src$$index;
  4271       int scale    = $src$$scale;
  4272       int displace = $src$$disp;
  4273       bool disp_is_oop = $src->disp_is_oop(); // disp-as-oop when working with static globals
  4274       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
  4276     cbuf.set_insts_mark();            // Mark start of MOVSD in case $mem has an oop
  4277     { // MOVSD $mem,$tmp ! atomic long store
  4278       emit_opcode(cbuf,0xF2);
  4279       emit_opcode(cbuf,0x0F);
  4280       emit_opcode(cbuf,0x11);
  4281       int base     = $mem$$base;
  4282       int index    = $mem$$index;
  4283       int scale    = $mem$$scale;
  4284       int displace = $mem$$disp;
  4285       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
  4286       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
  4288   %}
  4290   enc_class enc_storeLX_reg_volatile( memory mem, eRegL src, regXD tmp, regXD tmp2) %{
  4291     { // MOVD $tmp,$src.lo
  4292       emit_opcode(cbuf,0x66);
  4293       emit_opcode(cbuf,0x0F);
  4294       emit_opcode(cbuf,0x6E);
  4295       emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);
  4297     { // MOVD $tmp2,$src.hi
  4298       emit_opcode(cbuf,0x66);
  4299       emit_opcode(cbuf,0x0F);
  4300       emit_opcode(cbuf,0x6E);
  4301       emit_rm(cbuf, 0x3, $tmp2$$reg, HIGH_FROM_LOW($src$$reg));
  4303     { // PUNPCKLDQ $tmp,$tmp2
  4304       emit_opcode(cbuf,0x66);
  4305       emit_opcode(cbuf,0x0F);
  4306       emit_opcode(cbuf,0x62);
  4307       emit_rm(cbuf, 0x3, $tmp$$reg, $tmp2$$reg);
  4309     cbuf.set_insts_mark();            // Mark start of MOVSD in case $mem has an oop
  4310     { // MOVSD $mem,$tmp ! atomic long store
  4311       emit_opcode(cbuf,0xF2);
  4312       emit_opcode(cbuf,0x0F);
  4313       emit_opcode(cbuf,0x11);
  4314       int base     = $mem$$base;
  4315       int index    = $mem$$index;
  4316       int scale    = $mem$$scale;
  4317       int displace = $mem$$disp;
  4318       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
  4319       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
  4321   %}
  4323   // Safepoint Poll.  This polls the safepoint page, and causes an
  4324   // exception if it is not readable. Unfortunately, it kills the condition code
  4325   // in the process
  4326   // We current use TESTL [spp],EDI
  4327   // A better choice might be TESTB [spp + pagesize() - CacheLineSize()],0
  4329   enc_class Safepoint_Poll() %{
  4330     cbuf.relocate(cbuf.insts_mark(), relocInfo::poll_type, 0);
  4331     emit_opcode(cbuf,0x85);
  4332     emit_rm (cbuf, 0x0, 0x7, 0x5);
  4333     emit_d32(cbuf, (intptr_t)os::get_polling_page());
  4334   %}
  4335 %}
  4338 //----------FRAME--------------------------------------------------------------
  4339 // Definition of frame structure and management information.
  4340 //
  4341 //  S T A C K   L A Y O U T    Allocators stack-slot number
  4342 //                             |   (to get allocators register number
  4343 //  G  Owned by    |        |  v    add OptoReg::stack0())
  4344 //  r   CALLER     |        |
  4345 //  o     |        +--------+      pad to even-align allocators stack-slot
  4346 //  w     V        |  pad0  |        numbers; owned by CALLER
  4347 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
  4348 //  h     ^        |   in   |  5
  4349 //        |        |  args  |  4   Holes in incoming args owned by SELF
  4350 //  |     |        |        |  3
  4351 //  |     |        +--------+
  4352 //  V     |        | old out|      Empty on Intel, window on Sparc
  4353 //        |    old |preserve|      Must be even aligned.
  4354 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
  4355 //        |        |   in   |  3   area for Intel ret address
  4356 //     Owned by    |preserve|      Empty on Sparc.
  4357 //       SELF      +--------+
  4358 //        |        |  pad2  |  2   pad to align old SP
  4359 //        |        +--------+  1
  4360 //        |        | locks  |  0
  4361 //        |        +--------+----> OptoReg::stack0(), even aligned
  4362 //        |        |  pad1  | 11   pad to align new SP
  4363 //        |        +--------+
  4364 //        |        |        | 10
  4365 //        |        | spills |  9   spills
  4366 //        V        |        |  8   (pad0 slot for callee)
  4367 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
  4368 //        ^        |  out   |  7
  4369 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
  4370 //     Owned by    +--------+
  4371 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
  4372 //        |    new |preserve|      Must be even-aligned.
  4373 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
  4374 //        |        |        |
  4375 //
  4376 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
  4377 //         known from SELF's arguments and the Java calling convention.
  4378 //         Region 6-7 is determined per call site.
  4379 // Note 2: If the calling convention leaves holes in the incoming argument
  4380 //         area, those holes are owned by SELF.  Holes in the outgoing area
  4381 //         are owned by the CALLEE.  Holes should not be nessecary in the
  4382 //         incoming area, as the Java calling convention is completely under
  4383 //         the control of the AD file.  Doubles can be sorted and packed to
  4384 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
  4385 //         varargs C calling conventions.
  4386 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
  4387 //         even aligned with pad0 as needed.
  4388 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
  4389 //         region 6-11 is even aligned; it may be padded out more so that
  4390 //         the region from SP to FP meets the minimum stack alignment.
  4392 frame %{
  4393   // What direction does stack grow in (assumed to be same for C & Java)
  4394   stack_direction(TOWARDS_LOW);
  4396   // These three registers define part of the calling convention
  4397   // between compiled code and the interpreter.
  4398   inline_cache_reg(EAX);                // Inline Cache Register
  4399   interpreter_method_oop_reg(EBX);      // Method Oop Register when calling interpreter
  4401   // Optional: name the operand used by cisc-spilling to access [stack_pointer + offset]
  4402   cisc_spilling_operand_name(indOffset32);
  4404   // Number of stack slots consumed by locking an object
  4405   sync_stack_slots(1);
  4407   // Compiled code's Frame Pointer
  4408   frame_pointer(ESP);
  4409   // Interpreter stores its frame pointer in a register which is
  4410   // stored to the stack by I2CAdaptors.
  4411   // I2CAdaptors convert from interpreted java to compiled java.
  4412   interpreter_frame_pointer(EBP);
  4414   // Stack alignment requirement
  4415   // Alignment size in bytes (128-bit -> 16 bytes)
  4416   stack_alignment(StackAlignmentInBytes);
  4418   // Number of stack slots between incoming argument block and the start of
  4419   // a new frame.  The PROLOG must add this many slots to the stack.  The
  4420   // EPILOG must remove this many slots.  Intel needs one slot for
  4421   // return address and one for rbp, (must save rbp)
  4422   in_preserve_stack_slots(2+VerifyStackAtCalls);
  4424   // Number of outgoing stack slots killed above the out_preserve_stack_slots
  4425   // for calls to C.  Supports the var-args backing area for register parms.
  4426   varargs_C_out_slots_killed(0);
  4428   // The after-PROLOG location of the return address.  Location of
  4429   // return address specifies a type (REG or STACK) and a number
  4430   // representing the register number (i.e. - use a register name) or
  4431   // stack slot.
  4432   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
  4433   // Otherwise, it is above the locks and verification slot and alignment word
  4434   return_addr(STACK - 1 +
  4435               round_to(1+VerifyStackAtCalls+
  4436               Compile::current()->fixed_slots(),
  4437               (StackAlignmentInBytes/wordSize)));
  4439   // Body of function which returns an integer array locating
  4440   // arguments either in registers or in stack slots.  Passed an array
  4441   // of ideal registers called "sig" and a "length" count.  Stack-slot
  4442   // offsets are based on outgoing arguments, i.e. a CALLER setting up
  4443   // arguments for a CALLEE.  Incoming stack arguments are
  4444   // automatically biased by the preserve_stack_slots field above.
  4445   calling_convention %{
  4446     // No difference between ingoing/outgoing just pass false
  4447     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
  4448   %}
  4451   // Body of function which returns an integer array locating
  4452   // arguments either in registers or in stack slots.  Passed an array
  4453   // of ideal registers called "sig" and a "length" count.  Stack-slot
  4454   // offsets are based on outgoing arguments, i.e. a CALLER setting up
  4455   // arguments for a CALLEE.  Incoming stack arguments are
  4456   // automatically biased by the preserve_stack_slots field above.
  4457   c_calling_convention %{
  4458     // This is obviously always outgoing
  4459     (void) SharedRuntime::c_calling_convention(sig_bt, regs, length);
  4460   %}
  4462   // Location of C & interpreter return values
  4463   c_return_value %{
  4464     assert( ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values" );
  4465     static int lo[Op_RegL+1] = { 0, 0, OptoReg::Bad, EAX_num,      EAX_num,      FPR1L_num,    FPR1L_num, EAX_num };
  4466     static int hi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, FPR1H_num, EDX_num };
  4468     // in SSE2+ mode we want to keep the FPU stack clean so pretend
  4469     // that C functions return float and double results in XMM0.
  4470     if( ideal_reg == Op_RegD && UseSSE>=2 )
  4471       return OptoRegPair(XMM0b_num,XMM0a_num);
  4472     if( ideal_reg == Op_RegF && UseSSE>=2 )
  4473       return OptoRegPair(OptoReg::Bad,XMM0a_num);
  4475     return OptoRegPair(hi[ideal_reg],lo[ideal_reg]);
  4476   %}
  4478   // Location of return values
  4479   return_value %{
  4480     assert( ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values" );
  4481     static int lo[Op_RegL+1] = { 0, 0, OptoReg::Bad, EAX_num,      EAX_num,      FPR1L_num,    FPR1L_num, EAX_num };
  4482     static int hi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, FPR1H_num, EDX_num };
  4483     if( ideal_reg == Op_RegD && UseSSE>=2 )
  4484       return OptoRegPair(XMM0b_num,XMM0a_num);
  4485     if( ideal_reg == Op_RegF && UseSSE>=1 )
  4486       return OptoRegPair(OptoReg::Bad,XMM0a_num);
  4487     return OptoRegPair(hi[ideal_reg],lo[ideal_reg]);
  4488   %}
  4490 %}
  4492 //----------ATTRIBUTES---------------------------------------------------------
  4493 //----------Operand Attributes-------------------------------------------------
  4494 op_attrib op_cost(0);        // Required cost attribute
  4496 //----------Instruction Attributes---------------------------------------------
  4497 ins_attrib ins_cost(100);       // Required cost attribute
  4498 ins_attrib ins_size(8);         // Required size attribute (in bits)
  4499 ins_attrib ins_pc_relative(0);  // Required PC Relative flag
  4500 ins_attrib ins_short_branch(0); // Required flag: is this instruction a
  4501                                 // non-matching short branch variant of some
  4502                                                             // long branch?
  4503 ins_attrib ins_alignment(1);    // Required alignment attribute (must be a power of 2)
  4504                                 // specifies the alignment that some part of the instruction (not
  4505                                 // necessarily the start) requires.  If > 1, a compute_padding()
  4506                                 // function must be provided for the instruction
  4508 //----------OPERANDS-----------------------------------------------------------
  4509 // Operand definitions must precede instruction definitions for correct parsing
  4510 // in the ADLC because operands constitute user defined types which are used in
  4511 // instruction definitions.
  4513 //----------Simple Operands----------------------------------------------------
  4514 // Immediate Operands
  4515 // Integer Immediate
  4516 operand immI() %{
  4517   match(ConI);
  4519   op_cost(10);
  4520   format %{ %}
  4521   interface(CONST_INTER);
  4522 %}
  4524 // Constant for test vs zero
  4525 operand immI0() %{
  4526   predicate(n->get_int() == 0);
  4527   match(ConI);
  4529   op_cost(0);
  4530   format %{ %}
  4531   interface(CONST_INTER);
  4532 %}
  4534 // Constant for increment
  4535 operand immI1() %{
  4536   predicate(n->get_int() == 1);
  4537   match(ConI);
  4539   op_cost(0);
  4540   format %{ %}
  4541   interface(CONST_INTER);
  4542 %}
  4544 // Constant for decrement
  4545 operand immI_M1() %{
  4546   predicate(n->get_int() == -1);
  4547   match(ConI);
  4549   op_cost(0);
  4550   format %{ %}
  4551   interface(CONST_INTER);
  4552 %}
  4554 // Valid scale values for addressing modes
  4555 operand immI2() %{
  4556   predicate(0 <= n->get_int() && (n->get_int() <= 3));
  4557   match(ConI);
  4559   format %{ %}
  4560   interface(CONST_INTER);
  4561 %}
  4563 operand immI8() %{
  4564   predicate((-128 <= n->get_int()) && (n->get_int() <= 127));
  4565   match(ConI);
  4567   op_cost(5);
  4568   format %{ %}
  4569   interface(CONST_INTER);
  4570 %}
  4572 operand immI16() %{
  4573   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
  4574   match(ConI);
  4576   op_cost(10);
  4577   format %{ %}
  4578   interface(CONST_INTER);
  4579 %}
  4581 // Constant for long shifts
  4582 operand immI_32() %{
  4583   predicate( n->get_int() == 32 );
  4584   match(ConI);
  4586   op_cost(0);
  4587   format %{ %}
  4588   interface(CONST_INTER);
  4589 %}
  4591 operand immI_1_31() %{
  4592   predicate( n->get_int() >= 1 && n->get_int() <= 31 );
  4593   match(ConI);
  4595   op_cost(0);
  4596   format %{ %}
  4597   interface(CONST_INTER);
  4598 %}
  4600 operand immI_32_63() %{
  4601   predicate( n->get_int() >= 32 && n->get_int() <= 63 );
  4602   match(ConI);
  4603   op_cost(0);
  4605   format %{ %}
  4606   interface(CONST_INTER);
  4607 %}
  4609 operand immI_1() %{
  4610   predicate( n->get_int() == 1 );
  4611   match(ConI);
  4613   op_cost(0);
  4614   format %{ %}
  4615   interface(CONST_INTER);
  4616 %}
  4618 operand immI_2() %{
  4619   predicate( n->get_int() == 2 );
  4620   match(ConI);
  4622   op_cost(0);
  4623   format %{ %}
  4624   interface(CONST_INTER);
  4625 %}
  4627 operand immI_3() %{
  4628   predicate( n->get_int() == 3 );
  4629   match(ConI);
  4631   op_cost(0);
  4632   format %{ %}
  4633   interface(CONST_INTER);
  4634 %}
  4636 // Pointer Immediate
  4637 operand immP() %{
  4638   match(ConP);
  4640   op_cost(10);
  4641   format %{ %}
  4642   interface(CONST_INTER);
  4643 %}
  4645 // NULL Pointer Immediate
  4646 operand immP0() %{
  4647   predicate( n->get_ptr() == 0 );
  4648   match(ConP);
  4649   op_cost(0);
  4651   format %{ %}
  4652   interface(CONST_INTER);
  4653 %}
  4655 // Long Immediate
  4656 operand immL() %{
  4657   match(ConL);
  4659   op_cost(20);
  4660   format %{ %}
  4661   interface(CONST_INTER);
  4662 %}
  4664 // Long Immediate zero
  4665 operand immL0() %{
  4666   predicate( n->get_long() == 0L );
  4667   match(ConL);
  4668   op_cost(0);
  4670   format %{ %}
  4671   interface(CONST_INTER);
  4672 %}
  4674 // Long Immediate zero
  4675 operand immL_M1() %{
  4676   predicate( n->get_long() == -1L );
  4677   match(ConL);
  4678   op_cost(0);
  4680   format %{ %}
  4681   interface(CONST_INTER);
  4682 %}
  4684 // Long immediate from 0 to 127.
  4685 // Used for a shorter form of long mul by 10.
  4686 operand immL_127() %{
  4687   predicate((0 <= n->get_long()) && (n->get_long() <= 127));
  4688   match(ConL);
  4689   op_cost(0);
  4691   format %{ %}
  4692   interface(CONST_INTER);
  4693 %}
  4695 // Long Immediate: low 32-bit mask
  4696 operand immL_32bits() %{
  4697   predicate(n->get_long() == 0xFFFFFFFFL);
  4698   match(ConL);
  4699   op_cost(0);
  4701   format %{ %}
  4702   interface(CONST_INTER);
  4703 %}
  4705 // Long Immediate: low 32-bit mask
  4706 operand immL32() %{
  4707   predicate(n->get_long() == (int)(n->get_long()));
  4708   match(ConL);
  4709   op_cost(20);
  4711   format %{ %}
  4712   interface(CONST_INTER);
  4713 %}
  4715 //Double Immediate zero
  4716 operand immD0() %{
  4717   // Do additional (and counter-intuitive) test against NaN to work around VC++
  4718   // bug that generates code such that NaNs compare equal to 0.0
  4719   predicate( UseSSE<=1 && n->getd() == 0.0 && !g_isnan(n->getd()) );
  4720   match(ConD);
  4722   op_cost(5);
  4723   format %{ %}
  4724   interface(CONST_INTER);
  4725 %}
  4727 // Double Immediate one
  4728 operand immD1() %{
  4729   predicate( UseSSE<=1 && n->getd() == 1.0 );
  4730   match(ConD);
  4732   op_cost(5);
  4733   format %{ %}
  4734   interface(CONST_INTER);
  4735 %}
  4737 // Double Immediate
  4738 operand immD() %{
  4739   predicate(UseSSE<=1);
  4740   match(ConD);
  4742   op_cost(5);
  4743   format %{ %}
  4744   interface(CONST_INTER);
  4745 %}
  4747 operand immXD() %{
  4748   predicate(UseSSE>=2);
  4749   match(ConD);
  4751   op_cost(5);
  4752   format %{ %}
  4753   interface(CONST_INTER);
  4754 %}
  4756 // Double Immediate zero
  4757 operand immXD0() %{
  4758   // Do additional (and counter-intuitive) test against NaN to work around VC++
  4759   // bug that generates code such that NaNs compare equal to 0.0 AND do not
  4760   // compare equal to -0.0.
  4761   predicate( UseSSE>=2 && jlong_cast(n->getd()) == 0 );
  4762   match(ConD);
  4764   format %{ %}
  4765   interface(CONST_INTER);
  4766 %}
  4768 // Float Immediate zero
  4769 operand immF0() %{
  4770   predicate(UseSSE == 0 && n->getf() == 0.0F);
  4771   match(ConF);
  4773   op_cost(5);
  4774   format %{ %}
  4775   interface(CONST_INTER);
  4776 %}
  4778 // Float Immediate one
  4779 operand immF1() %{
  4780   predicate(UseSSE == 0 && n->getf() == 1.0F);
  4781   match(ConF);
  4783   op_cost(5);
  4784   format %{ %}
  4785   interface(CONST_INTER);
  4786 %}
  4788 // Float Immediate
  4789 operand immF() %{
  4790   predicate( UseSSE == 0 );
  4791   match(ConF);
  4793   op_cost(5);
  4794   format %{ %}
  4795   interface(CONST_INTER);
  4796 %}
  4798 // Float Immediate
  4799 operand immXF() %{
  4800   predicate(UseSSE >= 1);
  4801   match(ConF);
  4803   op_cost(5);
  4804   format %{ %}
  4805   interface(CONST_INTER);
  4806 %}
  4808 // Float Immediate zero.  Zero and not -0.0
  4809 operand immXF0() %{
  4810   predicate( UseSSE >= 1 && jint_cast(n->getf()) == 0 );
  4811   match(ConF);
  4813   op_cost(5);
  4814   format %{ %}
  4815   interface(CONST_INTER);
  4816 %}
  4818 // Immediates for special shifts (sign extend)
  4820 // Constants for increment
  4821 operand immI_16() %{
  4822   predicate( n->get_int() == 16 );
  4823   match(ConI);
  4825   format %{ %}
  4826   interface(CONST_INTER);
  4827 %}
  4829 operand immI_24() %{
  4830   predicate( n->get_int() == 24 );
  4831   match(ConI);
  4833   format %{ %}
  4834   interface(CONST_INTER);
  4835 %}
  4837 // Constant for byte-wide masking
  4838 operand immI_255() %{
  4839   predicate( n->get_int() == 255 );
  4840   match(ConI);
  4842   format %{ %}
  4843   interface(CONST_INTER);
  4844 %}
  4846 // Constant for short-wide masking
  4847 operand immI_65535() %{
  4848   predicate(n->get_int() == 65535);
  4849   match(ConI);
  4851   format %{ %}
  4852   interface(CONST_INTER);
  4853 %}
  4855 // Register Operands
  4856 // Integer Register
  4857 operand eRegI() %{
  4858   constraint(ALLOC_IN_RC(e_reg));
  4859   match(RegI);
  4860   match(xRegI);
  4861   match(eAXRegI);
  4862   match(eBXRegI);
  4863   match(eCXRegI);
  4864   match(eDXRegI);
  4865   match(eDIRegI);
  4866   match(eSIRegI);
  4868   format %{ %}
  4869   interface(REG_INTER);
  4870 %}
  4872 // Subset of Integer Register
  4873 operand xRegI(eRegI reg) %{
  4874   constraint(ALLOC_IN_RC(x_reg));
  4875   match(reg);
  4876   match(eAXRegI);
  4877   match(eBXRegI);
  4878   match(eCXRegI);
  4879   match(eDXRegI);
  4881   format %{ %}
  4882   interface(REG_INTER);
  4883 %}
  4885 // Special Registers
  4886 operand eAXRegI(xRegI reg) %{
  4887   constraint(ALLOC_IN_RC(eax_reg));
  4888   match(reg);
  4889   match(eRegI);
  4891   format %{ "EAX" %}
  4892   interface(REG_INTER);
  4893 %}
  4895 // Special Registers
  4896 operand eBXRegI(xRegI reg) %{
  4897   constraint(ALLOC_IN_RC(ebx_reg));
  4898   match(reg);
  4899   match(eRegI);
  4901   format %{ "EBX" %}
  4902   interface(REG_INTER);
  4903 %}
  4905 operand eCXRegI(xRegI reg) %{
  4906   constraint(ALLOC_IN_RC(ecx_reg));
  4907   match(reg);
  4908   match(eRegI);
  4910   format %{ "ECX" %}
  4911   interface(REG_INTER);
  4912 %}
  4914 operand eDXRegI(xRegI reg) %{
  4915   constraint(ALLOC_IN_RC(edx_reg));
  4916   match(reg);
  4917   match(eRegI);
  4919   format %{ "EDX" %}
  4920   interface(REG_INTER);
  4921 %}
  4923 operand eDIRegI(xRegI reg) %{
  4924   constraint(ALLOC_IN_RC(edi_reg));
  4925   match(reg);
  4926   match(eRegI);
  4928   format %{ "EDI" %}
  4929   interface(REG_INTER);
  4930 %}
  4932 operand naxRegI() %{
  4933   constraint(ALLOC_IN_RC(nax_reg));
  4934   match(RegI);
  4935   match(eCXRegI);
  4936   match(eDXRegI);
  4937   match(eSIRegI);
  4938   match(eDIRegI);
  4940   format %{ %}
  4941   interface(REG_INTER);
  4942 %}
  4944 operand nadxRegI() %{
  4945   constraint(ALLOC_IN_RC(nadx_reg));
  4946   match(RegI);
  4947   match(eBXRegI);
  4948   match(eCXRegI);
  4949   match(eSIRegI);
  4950   match(eDIRegI);
  4952   format %{ %}
  4953   interface(REG_INTER);
  4954 %}
  4956 operand ncxRegI() %{
  4957   constraint(ALLOC_IN_RC(ncx_reg));
  4958   match(RegI);
  4959   match(eAXRegI);
  4960   match(eDXRegI);
  4961   match(eSIRegI);
  4962   match(eDIRegI);
  4964   format %{ %}
  4965   interface(REG_INTER);
  4966 %}
  4968 // // This operand was used by cmpFastUnlock, but conflicted with 'object' reg
  4969 // //
  4970 operand eSIRegI(xRegI reg) %{
  4971    constraint(ALLOC_IN_RC(esi_reg));
  4972    match(reg);
  4973    match(eRegI);
  4975    format %{ "ESI" %}
  4976    interface(REG_INTER);
  4977 %}
  4979 // Pointer Register
  4980 operand anyRegP() %{
  4981   constraint(ALLOC_IN_RC(any_reg));
  4982   match(RegP);
  4983   match(eAXRegP);
  4984   match(eBXRegP);
  4985   match(eCXRegP);
  4986   match(eDIRegP);
  4987   match(eRegP);
  4989   format %{ %}
  4990   interface(REG_INTER);
  4991 %}
  4993 operand eRegP() %{
  4994   constraint(ALLOC_IN_RC(e_reg));
  4995   match(RegP);
  4996   match(eAXRegP);
  4997   match(eBXRegP);
  4998   match(eCXRegP);
  4999   match(eDIRegP);
  5001   format %{ %}
  5002   interface(REG_INTER);
  5003 %}
  5005 // On windows95, EBP is not safe to use for implicit null tests.
  5006 operand eRegP_no_EBP() %{
  5007   constraint(ALLOC_IN_RC(e_reg_no_rbp));
  5008   match(RegP);
  5009   match(eAXRegP);
  5010   match(eBXRegP);
  5011   match(eCXRegP);
  5012   match(eDIRegP);
  5014   op_cost(100);
  5015   format %{ %}
  5016   interface(REG_INTER);
  5017 %}
  5019 operand naxRegP() %{
  5020   constraint(ALLOC_IN_RC(nax_reg));
  5021   match(RegP);
  5022   match(eBXRegP);
  5023   match(eDXRegP);
  5024   match(eCXRegP);
  5025   match(eSIRegP);
  5026   match(eDIRegP);
  5028   format %{ %}
  5029   interface(REG_INTER);
  5030 %}
  5032 operand nabxRegP() %{
  5033   constraint(ALLOC_IN_RC(nabx_reg));
  5034   match(RegP);
  5035   match(eCXRegP);
  5036   match(eDXRegP);
  5037   match(eSIRegP);
  5038   match(eDIRegP);
  5040   format %{ %}
  5041   interface(REG_INTER);
  5042 %}
  5044 operand pRegP() %{
  5045   constraint(ALLOC_IN_RC(p_reg));
  5046   match(RegP);
  5047   match(eBXRegP);
  5048   match(eDXRegP);
  5049   match(eSIRegP);
  5050   match(eDIRegP);
  5052   format %{ %}
  5053   interface(REG_INTER);
  5054 %}
  5056 // Special Registers
  5057 // Return a pointer value
  5058 operand eAXRegP(eRegP reg) %{
  5059   constraint(ALLOC_IN_RC(eax_reg));
  5060   match(reg);
  5061   format %{ "EAX" %}
  5062   interface(REG_INTER);
  5063 %}
  5065 // Used in AtomicAdd
  5066 operand eBXRegP(eRegP reg) %{
  5067   constraint(ALLOC_IN_RC(ebx_reg));
  5068   match(reg);
  5069   format %{ "EBX" %}
  5070   interface(REG_INTER);
  5071 %}
  5073 // Tail-call (interprocedural jump) to interpreter
  5074 operand eCXRegP(eRegP reg) %{
  5075   constraint(ALLOC_IN_RC(ecx_reg));
  5076   match(reg);
  5077   format %{ "ECX" %}
  5078   interface(REG_INTER);
  5079 %}
  5081 operand eSIRegP(eRegP reg) %{
  5082   constraint(ALLOC_IN_RC(esi_reg));
  5083   match(reg);
  5084   format %{ "ESI" %}
  5085   interface(REG_INTER);
  5086 %}
  5088 // Used in rep stosw
  5089 operand eDIRegP(eRegP reg) %{
  5090   constraint(ALLOC_IN_RC(edi_reg));
  5091   match(reg);
  5092   format %{ "EDI" %}
  5093   interface(REG_INTER);
  5094 %}
  5096 operand eBPRegP() %{
  5097   constraint(ALLOC_IN_RC(ebp_reg));
  5098   match(RegP);
  5099   format %{ "EBP" %}
  5100   interface(REG_INTER);
  5101 %}
  5103 operand eRegL() %{
  5104   constraint(ALLOC_IN_RC(long_reg));
  5105   match(RegL);
  5106   match(eADXRegL);
  5108   format %{ %}
  5109   interface(REG_INTER);
  5110 %}
  5112 operand eADXRegL( eRegL reg ) %{
  5113   constraint(ALLOC_IN_RC(eadx_reg));
  5114   match(reg);
  5116   format %{ "EDX:EAX" %}
  5117   interface(REG_INTER);
  5118 %}
  5120 operand eBCXRegL( eRegL reg ) %{
  5121   constraint(ALLOC_IN_RC(ebcx_reg));
  5122   match(reg);
  5124   format %{ "EBX:ECX" %}
  5125   interface(REG_INTER);
  5126 %}
  5128 // Special case for integer high multiply
  5129 operand eADXRegL_low_only() %{
  5130   constraint(ALLOC_IN_RC(eadx_reg));
  5131   match(RegL);
  5133   format %{ "EAX" %}
  5134   interface(REG_INTER);
  5135 %}
  5137 // Flags register, used as output of compare instructions
  5138 operand eFlagsReg() %{
  5139   constraint(ALLOC_IN_RC(int_flags));
  5140   match(RegFlags);
  5142   format %{ "EFLAGS" %}
  5143   interface(REG_INTER);
  5144 %}
  5146 // Flags register, used as output of FLOATING POINT compare instructions
  5147 operand eFlagsRegU() %{
  5148   constraint(ALLOC_IN_RC(int_flags));
  5149   match(RegFlags);
  5151   format %{ "EFLAGS_U" %}
  5152   interface(REG_INTER);
  5153 %}
  5155 operand eFlagsRegUCF() %{
  5156   constraint(ALLOC_IN_RC(int_flags));
  5157   match(RegFlags);
  5158   predicate(false);
  5160   format %{ "EFLAGS_U_CF" %}
  5161   interface(REG_INTER);
  5162 %}
  5164 // Condition Code Register used by long compare
  5165 operand flagsReg_long_LTGE() %{
  5166   constraint(ALLOC_IN_RC(int_flags));
  5167   match(RegFlags);
  5168   format %{ "FLAGS_LTGE" %}
  5169   interface(REG_INTER);
  5170 %}
  5171 operand flagsReg_long_EQNE() %{
  5172   constraint(ALLOC_IN_RC(int_flags));
  5173   match(RegFlags);
  5174   format %{ "FLAGS_EQNE" %}
  5175   interface(REG_INTER);
  5176 %}
  5177 operand flagsReg_long_LEGT() %{
  5178   constraint(ALLOC_IN_RC(int_flags));
  5179   match(RegFlags);
  5180   format %{ "FLAGS_LEGT" %}
  5181   interface(REG_INTER);
  5182 %}
  5184 // Float register operands
  5185 operand regD() %{
  5186   predicate( UseSSE < 2 );
  5187   constraint(ALLOC_IN_RC(dbl_reg));
  5188   match(RegD);
  5189   match(regDPR1);
  5190   match(regDPR2);
  5191   format %{ %}
  5192   interface(REG_INTER);
  5193 %}
  5195 operand regDPR1(regD reg) %{
  5196   predicate( UseSSE < 2 );
  5197   constraint(ALLOC_IN_RC(dbl_reg0));
  5198   match(reg);
  5199   format %{ "FPR1" %}
  5200   interface(REG_INTER);
  5201 %}
  5203 operand regDPR2(regD reg) %{
  5204   predicate( UseSSE < 2 );
  5205   constraint(ALLOC_IN_RC(dbl_reg1));
  5206   match(reg);
  5207   format %{ "FPR2" %}
  5208   interface(REG_INTER);
  5209 %}
  5211 operand regnotDPR1(regD reg) %{
  5212   predicate( UseSSE < 2 );
  5213   constraint(ALLOC_IN_RC(dbl_notreg0));
  5214   match(reg);
  5215   format %{ %}
  5216   interface(REG_INTER);
  5217 %}
  5219 // XMM Double register operands
  5220 operand regXD() %{
  5221   predicate( UseSSE>=2 );
  5222   constraint(ALLOC_IN_RC(xdb_reg));
  5223   match(RegD);
  5224   match(regXD6);
  5225   match(regXD7);
  5226   format %{ %}
  5227   interface(REG_INTER);
  5228 %}
  5230 // XMM6 double register operands
  5231 operand regXD6(regXD reg) %{
  5232   predicate( UseSSE>=2 );
  5233   constraint(ALLOC_IN_RC(xdb_reg6));
  5234   match(reg);
  5235   format %{ "XMM6" %}
  5236   interface(REG_INTER);
  5237 %}
  5239 // XMM7 double register operands
  5240 operand regXD7(regXD reg) %{
  5241   predicate( UseSSE>=2 );
  5242   constraint(ALLOC_IN_RC(xdb_reg7));
  5243   match(reg);
  5244   format %{ "XMM7" %}
  5245   interface(REG_INTER);
  5246 %}
  5248 // Float register operands
  5249 operand regF() %{
  5250   predicate( UseSSE < 2 );
  5251   constraint(ALLOC_IN_RC(flt_reg));
  5252   match(RegF);
  5253   match(regFPR1);
  5254   format %{ %}
  5255   interface(REG_INTER);
  5256 %}
  5258 // Float register operands
  5259 operand regFPR1(regF reg) %{
  5260   predicate( UseSSE < 2 );
  5261   constraint(ALLOC_IN_RC(flt_reg0));
  5262   match(reg);
  5263   format %{ "FPR1" %}
  5264   interface(REG_INTER);
  5265 %}
  5267 // XMM register operands
  5268 operand regX() %{
  5269   predicate( UseSSE>=1 );
  5270   constraint(ALLOC_IN_RC(xmm_reg));
  5271   match(RegF);
  5272   format %{ %}
  5273   interface(REG_INTER);
  5274 %}
  5277 //----------Memory Operands----------------------------------------------------
  5278 // Direct Memory Operand
  5279 operand direct(immP addr) %{
  5280   match(addr);
  5282   format %{ "[$addr]" %}
  5283   interface(MEMORY_INTER) %{
  5284     base(0xFFFFFFFF);
  5285     index(0x4);
  5286     scale(0x0);
  5287     disp($addr);
  5288   %}
  5289 %}
  5291 // Indirect Memory Operand
  5292 operand indirect(eRegP reg) %{
  5293   constraint(ALLOC_IN_RC(e_reg));
  5294   match(reg);
  5296   format %{ "[$reg]" %}
  5297   interface(MEMORY_INTER) %{
  5298     base($reg);
  5299     index(0x4);
  5300     scale(0x0);
  5301     disp(0x0);
  5302   %}
  5303 %}
  5305 // Indirect Memory Plus Short Offset Operand
  5306 operand indOffset8(eRegP reg, immI8 off) %{
  5307   match(AddP reg off);
  5309   format %{ "[$reg + $off]" %}
  5310   interface(MEMORY_INTER) %{
  5311     base($reg);
  5312     index(0x4);
  5313     scale(0x0);
  5314     disp($off);
  5315   %}
  5316 %}
  5318 // Indirect Memory Plus Long Offset Operand
  5319 operand indOffset32(eRegP reg, immI off) %{
  5320   match(AddP reg off);
  5322   format %{ "[$reg + $off]" %}
  5323   interface(MEMORY_INTER) %{
  5324     base($reg);
  5325     index(0x4);
  5326     scale(0x0);
  5327     disp($off);
  5328   %}
  5329 %}
  5331 // Indirect Memory Plus Long Offset Operand
  5332 operand indOffset32X(eRegI reg, immP off) %{
  5333   match(AddP off reg);
  5335   format %{ "[$reg + $off]" %}
  5336   interface(MEMORY_INTER) %{
  5337     base($reg);
  5338     index(0x4);
  5339     scale(0x0);
  5340     disp($off);
  5341   %}
  5342 %}
  5344 // Indirect Memory Plus Index Register Plus Offset Operand
  5345 operand indIndexOffset(eRegP reg, eRegI ireg, immI off) %{
  5346   match(AddP (AddP reg ireg) off);
  5348   op_cost(10);
  5349   format %{"[$reg + $off + $ireg]" %}
  5350   interface(MEMORY_INTER) %{
  5351     base($reg);
  5352     index($ireg);
  5353     scale(0x0);
  5354     disp($off);
  5355   %}
  5356 %}
  5358 // Indirect Memory Plus Index Register Plus Offset Operand
  5359 operand indIndex(eRegP reg, eRegI ireg) %{
  5360   match(AddP reg ireg);
  5362   op_cost(10);
  5363   format %{"[$reg + $ireg]" %}
  5364   interface(MEMORY_INTER) %{
  5365     base($reg);
  5366     index($ireg);
  5367     scale(0x0);
  5368     disp(0x0);
  5369   %}
  5370 %}
  5372 // // -------------------------------------------------------------------------
  5373 // // 486 architecture doesn't support "scale * index + offset" with out a base
  5374 // // -------------------------------------------------------------------------
  5375 // // Scaled Memory Operands
  5376 // // Indirect Memory Times Scale Plus Offset Operand
  5377 // operand indScaleOffset(immP off, eRegI ireg, immI2 scale) %{
  5378 //   match(AddP off (LShiftI ireg scale));
  5379 //
  5380 //   op_cost(10);
  5381 //   format %{"[$off + $ireg << $scale]" %}
  5382 //   interface(MEMORY_INTER) %{
  5383 //     base(0x4);
  5384 //     index($ireg);
  5385 //     scale($scale);
  5386 //     disp($off);
  5387 //   %}
  5388 // %}
  5390 // Indirect Memory Times Scale Plus Index Register
  5391 operand indIndexScale(eRegP reg, eRegI ireg, immI2 scale) %{
  5392   match(AddP reg (LShiftI ireg scale));
  5394   op_cost(10);
  5395   format %{"[$reg + $ireg << $scale]" %}
  5396   interface(MEMORY_INTER) %{
  5397     base($reg);
  5398     index($ireg);
  5399     scale($scale);
  5400     disp(0x0);
  5401   %}
  5402 %}
  5404 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
  5405 operand indIndexScaleOffset(eRegP reg, immI off, eRegI ireg, immI2 scale) %{
  5406   match(AddP (AddP reg (LShiftI ireg scale)) off);
  5408   op_cost(10);
  5409   format %{"[$reg + $off + $ireg << $scale]" %}
  5410   interface(MEMORY_INTER) %{
  5411     base($reg);
  5412     index($ireg);
  5413     scale($scale);
  5414     disp($off);
  5415   %}
  5416 %}
  5418 //----------Load Long Memory Operands------------------------------------------
  5419 // The load-long idiom will use it's address expression again after loading
  5420 // the first word of the long.  If the load-long destination overlaps with
  5421 // registers used in the addressing expression, the 2nd half will be loaded
  5422 // from a clobbered address.  Fix this by requiring that load-long use
  5423 // address registers that do not overlap with the load-long target.
  5425 // load-long support
  5426 operand load_long_RegP() %{
  5427   constraint(ALLOC_IN_RC(esi_reg));
  5428   match(RegP);
  5429   match(eSIRegP);
  5430   op_cost(100);
  5431   format %{  %}
  5432   interface(REG_INTER);
  5433 %}
  5435 // Indirect Memory Operand Long
  5436 operand load_long_indirect(load_long_RegP reg) %{
  5437   constraint(ALLOC_IN_RC(esi_reg));
  5438   match(reg);
  5440   format %{ "[$reg]" %}
  5441   interface(MEMORY_INTER) %{
  5442     base($reg);
  5443     index(0x4);
  5444     scale(0x0);
  5445     disp(0x0);
  5446   %}
  5447 %}
  5449 // Indirect Memory Plus Long Offset Operand
  5450 operand load_long_indOffset32(load_long_RegP reg, immI off) %{
  5451   match(AddP reg off);
  5453   format %{ "[$reg + $off]" %}
  5454   interface(MEMORY_INTER) %{
  5455     base($reg);
  5456     index(0x4);
  5457     scale(0x0);
  5458     disp($off);
  5459   %}
  5460 %}
  5462 opclass load_long_memory(load_long_indirect, load_long_indOffset32);
  5465 //----------Special Memory Operands--------------------------------------------
  5466 // Stack Slot Operand - This operand is used for loading and storing temporary
  5467 //                      values on the stack where a match requires a value to
  5468 //                      flow through memory.
  5469 operand stackSlotP(sRegP reg) %{
  5470   constraint(ALLOC_IN_RC(stack_slots));
  5471   // No match rule because this operand is only generated in matching
  5472   format %{ "[$reg]" %}
  5473   interface(MEMORY_INTER) %{
  5474     base(0x4);   // ESP
  5475     index(0x4);  // No Index
  5476     scale(0x0);  // No Scale
  5477     disp($reg);  // Stack Offset
  5478   %}
  5479 %}
  5481 operand stackSlotI(sRegI reg) %{
  5482   constraint(ALLOC_IN_RC(stack_slots));
  5483   // No match rule because this operand is only generated in matching
  5484   format %{ "[$reg]" %}
  5485   interface(MEMORY_INTER) %{
  5486     base(0x4);   // ESP
  5487     index(0x4);  // No Index
  5488     scale(0x0);  // No Scale
  5489     disp($reg);  // Stack Offset
  5490   %}
  5491 %}
  5493 operand stackSlotF(sRegF reg) %{
  5494   constraint(ALLOC_IN_RC(stack_slots));
  5495   // No match rule because this operand is only generated in matching
  5496   format %{ "[$reg]" %}
  5497   interface(MEMORY_INTER) %{
  5498     base(0x4);   // ESP
  5499     index(0x4);  // No Index
  5500     scale(0x0);  // No Scale
  5501     disp($reg);  // Stack Offset
  5502   %}
  5503 %}
  5505 operand stackSlotD(sRegD reg) %{
  5506   constraint(ALLOC_IN_RC(stack_slots));
  5507   // No match rule because this operand is only generated in matching
  5508   format %{ "[$reg]" %}
  5509   interface(MEMORY_INTER) %{
  5510     base(0x4);   // ESP
  5511     index(0x4);  // No Index
  5512     scale(0x0);  // No Scale
  5513     disp($reg);  // Stack Offset
  5514   %}
  5515 %}
  5517 operand stackSlotL(sRegL reg) %{
  5518   constraint(ALLOC_IN_RC(stack_slots));
  5519   // No match rule because this operand is only generated in matching
  5520   format %{ "[$reg]" %}
  5521   interface(MEMORY_INTER) %{
  5522     base(0x4);   // ESP
  5523     index(0x4);  // No Index
  5524     scale(0x0);  // No Scale
  5525     disp($reg);  // Stack Offset
  5526   %}
  5527 %}
  5529 //----------Memory Operands - Win95 Implicit Null Variants----------------
  5530 // Indirect Memory Operand
  5531 operand indirect_win95_safe(eRegP_no_EBP reg)
  5532 %{
  5533   constraint(ALLOC_IN_RC(e_reg));
  5534   match(reg);
  5536   op_cost(100);
  5537   format %{ "[$reg]" %}
  5538   interface(MEMORY_INTER) %{
  5539     base($reg);
  5540     index(0x4);
  5541     scale(0x0);
  5542     disp(0x0);
  5543   %}
  5544 %}
  5546 // Indirect Memory Plus Short Offset Operand
  5547 operand indOffset8_win95_safe(eRegP_no_EBP reg, immI8 off)
  5548 %{
  5549   match(AddP reg off);
  5551   op_cost(100);
  5552   format %{ "[$reg + $off]" %}
  5553   interface(MEMORY_INTER) %{
  5554     base($reg);
  5555     index(0x4);
  5556     scale(0x0);
  5557     disp($off);
  5558   %}
  5559 %}
  5561 // Indirect Memory Plus Long Offset Operand
  5562 operand indOffset32_win95_safe(eRegP_no_EBP reg, immI off)
  5563 %{
  5564   match(AddP reg off);
  5566   op_cost(100);
  5567   format %{ "[$reg + $off]" %}
  5568   interface(MEMORY_INTER) %{
  5569     base($reg);
  5570     index(0x4);
  5571     scale(0x0);
  5572     disp($off);
  5573   %}
  5574 %}
  5576 // Indirect Memory Plus Index Register Plus Offset Operand
  5577 operand indIndexOffset_win95_safe(eRegP_no_EBP reg, eRegI ireg, immI off)
  5578 %{
  5579   match(AddP (AddP reg ireg) off);
  5581   op_cost(100);
  5582   format %{"[$reg + $off + $ireg]" %}
  5583   interface(MEMORY_INTER) %{
  5584     base($reg);
  5585     index($ireg);
  5586     scale(0x0);
  5587     disp($off);
  5588   %}
  5589 %}
  5591 // Indirect Memory Times Scale Plus Index Register
  5592 operand indIndexScale_win95_safe(eRegP_no_EBP reg, eRegI ireg, immI2 scale)
  5593 %{
  5594   match(AddP reg (LShiftI ireg scale));
  5596   op_cost(100);
  5597   format %{"[$reg + $ireg << $scale]" %}
  5598   interface(MEMORY_INTER) %{
  5599     base($reg);
  5600     index($ireg);
  5601     scale($scale);
  5602     disp(0x0);
  5603   %}
  5604 %}
  5606 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
  5607 operand indIndexScaleOffset_win95_safe(eRegP_no_EBP reg, immI off, eRegI ireg, immI2 scale)
  5608 %{
  5609   match(AddP (AddP reg (LShiftI ireg scale)) off);
  5611   op_cost(100);
  5612   format %{"[$reg + $off + $ireg << $scale]" %}
  5613   interface(MEMORY_INTER) %{
  5614     base($reg);
  5615     index($ireg);
  5616     scale($scale);
  5617     disp($off);
  5618   %}
  5619 %}
  5621 //----------Conditional Branch Operands----------------------------------------
  5622 // Comparison Op  - This is the operation of the comparison, and is limited to
  5623 //                  the following set of codes:
  5624 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
  5625 //
  5626 // Other attributes of the comparison, such as unsignedness, are specified
  5627 // by the comparison instruction that sets a condition code flags register.
  5628 // That result is represented by a flags operand whose subtype is appropriate
  5629 // to the unsignedness (etc.) of the comparison.
  5630 //
  5631 // Later, the instruction which matches both the Comparison Op (a Bool) and
  5632 // the flags (produced by the Cmp) specifies the coding of the comparison op
  5633 // by matching a specific subtype of Bool operand below, such as cmpOpU.
  5635 // Comparision Code
  5636 operand cmpOp() %{
  5637   match(Bool);
  5639   format %{ "" %}
  5640   interface(COND_INTER) %{
  5641     equal(0x4, "e");
  5642     not_equal(0x5, "ne");
  5643     less(0xC, "l");
  5644     greater_equal(0xD, "ge");
  5645     less_equal(0xE, "le");
  5646     greater(0xF, "g");
  5647   %}
  5648 %}
  5650 // Comparison Code, unsigned compare.  Used by FP also, with
  5651 // C2 (unordered) turned into GT or LT already.  The other bits
  5652 // C0 and C3 are turned into Carry & Zero flags.
  5653 operand cmpOpU() %{
  5654   match(Bool);
  5656   format %{ "" %}
  5657   interface(COND_INTER) %{
  5658     equal(0x4, "e");
  5659     not_equal(0x5, "ne");
  5660     less(0x2, "b");
  5661     greater_equal(0x3, "nb");
  5662     less_equal(0x6, "be");
  5663     greater(0x7, "nbe");
  5664   %}
  5665 %}
  5667 // Floating comparisons that don't require any fixup for the unordered case
  5668 operand cmpOpUCF() %{
  5669   match(Bool);
  5670   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
  5671             n->as_Bool()->_test._test == BoolTest::ge ||
  5672             n->as_Bool()->_test._test == BoolTest::le ||
  5673             n->as_Bool()->_test._test == BoolTest::gt);
  5674   format %{ "" %}
  5675   interface(COND_INTER) %{
  5676     equal(0x4, "e");
  5677     not_equal(0x5, "ne");
  5678     less(0x2, "b");
  5679     greater_equal(0x3, "nb");
  5680     less_equal(0x6, "be");
  5681     greater(0x7, "nbe");
  5682   %}
  5683 %}
  5686 // Floating comparisons that can be fixed up with extra conditional jumps
  5687 operand cmpOpUCF2() %{
  5688   match(Bool);
  5689   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
  5690             n->as_Bool()->_test._test == BoolTest::eq);
  5691   format %{ "" %}
  5692   interface(COND_INTER) %{
  5693     equal(0x4, "e");
  5694     not_equal(0x5, "ne");
  5695     less(0x2, "b");
  5696     greater_equal(0x3, "nb");
  5697     less_equal(0x6, "be");
  5698     greater(0x7, "nbe");
  5699   %}
  5700 %}
  5702 // Comparison Code for FP conditional move
  5703 operand cmpOp_fcmov() %{
  5704   match(Bool);
  5706   format %{ "" %}
  5707   interface(COND_INTER) %{
  5708     equal        (0x0C8);
  5709     not_equal    (0x1C8);
  5710     less         (0x0C0);
  5711     greater_equal(0x1C0);
  5712     less_equal   (0x0D0);
  5713     greater      (0x1D0);
  5714   %}
  5715 %}
  5717 // Comparision Code used in long compares
  5718 operand cmpOp_commute() %{
  5719   match(Bool);
  5721   format %{ "" %}
  5722   interface(COND_INTER) %{
  5723     equal(0x4, "e");
  5724     not_equal(0x5, "ne");
  5725     less(0xF, "g");
  5726     greater_equal(0xE, "le");
  5727     less_equal(0xD, "ge");
  5728     greater(0xC, "l");
  5729   %}
  5730 %}
  5732 //----------OPERAND CLASSES----------------------------------------------------
  5733 // Operand Classes are groups of operands that are used as to simplify
  5734 // instruction definitions by not requiring the AD writer to specify separate
  5735 // instructions for every form of operand when the instruction accepts
  5736 // multiple operand types with the same basic encoding and format.  The classic
  5737 // case of this is memory operands.
  5739 opclass memory(direct, indirect, indOffset8, indOffset32, indOffset32X, indIndexOffset,
  5740                indIndex, indIndexScale, indIndexScaleOffset);
  5742 // Long memory operations are encoded in 2 instructions and a +4 offset.
  5743 // This means some kind of offset is always required and you cannot use
  5744 // an oop as the offset (done when working on static globals).
  5745 opclass long_memory(direct, indirect, indOffset8, indOffset32, indIndexOffset,
  5746                     indIndex, indIndexScale, indIndexScaleOffset);
  5749 //----------PIPELINE-----------------------------------------------------------
  5750 // Rules which define the behavior of the target architectures pipeline.
  5751 pipeline %{
  5753 //----------ATTRIBUTES---------------------------------------------------------
  5754 attributes %{
  5755   variable_size_instructions;        // Fixed size instructions
  5756   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
  5757   instruction_unit_size = 1;         // An instruction is 1 bytes long
  5758   instruction_fetch_unit_size = 16;  // The processor fetches one line
  5759   instruction_fetch_units = 1;       // of 16 bytes
  5761   // List of nop instructions
  5762   nops( MachNop );
  5763 %}
  5765 //----------RESOURCES----------------------------------------------------------
  5766 // Resources are the functional units available to the machine
  5768 // Generic P2/P3 pipeline
  5769 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
  5770 // 3 instructions decoded per cycle.
  5771 // 2 load/store ops per cycle, 1 branch, 1 FPU,
  5772 // 2 ALU op, only ALU0 handles mul/div instructions.
  5773 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
  5774            MS0, MS1, MEM = MS0 | MS1,
  5775            BR, FPU,
  5776            ALU0, ALU1, ALU = ALU0 | ALU1 );
  5778 //----------PIPELINE DESCRIPTION-----------------------------------------------
  5779 // Pipeline Description specifies the stages in the machine's pipeline
  5781 // Generic P2/P3 pipeline
  5782 pipe_desc(S0, S1, S2, S3, S4, S5);
  5784 //----------PIPELINE CLASSES---------------------------------------------------
  5785 // Pipeline Classes describe the stages in which input and output are
  5786 // referenced by the hardware pipeline.
  5788 // Naming convention: ialu or fpu
  5789 // Then: _reg
  5790 // Then: _reg if there is a 2nd register
  5791 // Then: _long if it's a pair of instructions implementing a long
  5792 // Then: _fat if it requires the big decoder
  5793 //   Or: _mem if it requires the big decoder and a memory unit.
  5795 // Integer ALU reg operation
  5796 pipe_class ialu_reg(eRegI dst) %{
  5797     single_instruction;
  5798     dst    : S4(write);
  5799     dst    : S3(read);
  5800     DECODE : S0;        // any decoder
  5801     ALU    : S3;        // any alu
  5802 %}
  5804 // Long ALU reg operation
  5805 pipe_class ialu_reg_long(eRegL dst) %{
  5806     instruction_count(2);
  5807     dst    : S4(write);
  5808     dst    : S3(read);
  5809     DECODE : S0(2);     // any 2 decoders
  5810     ALU    : S3(2);     // both alus
  5811 %}
  5813 // Integer ALU reg operation using big decoder
  5814 pipe_class ialu_reg_fat(eRegI dst) %{
  5815     single_instruction;
  5816     dst    : S4(write);
  5817     dst    : S3(read);
  5818     D0     : S0;        // big decoder only
  5819     ALU    : S3;        // any alu
  5820 %}
  5822 // Long ALU reg operation using big decoder
  5823 pipe_class ialu_reg_long_fat(eRegL dst) %{
  5824     instruction_count(2);
  5825     dst    : S4(write);
  5826     dst    : S3(read);
  5827     D0     : S0(2);     // big decoder only; twice
  5828     ALU    : S3(2);     // any 2 alus
  5829 %}
  5831 // Integer ALU reg-reg operation
  5832 pipe_class ialu_reg_reg(eRegI dst, eRegI src) %{
  5833     single_instruction;
  5834     dst    : S4(write);
  5835     src    : S3(read);
  5836     DECODE : S0;        // any decoder
  5837     ALU    : S3;        // any alu
  5838 %}
  5840 // Long ALU reg-reg operation
  5841 pipe_class ialu_reg_reg_long(eRegL dst, eRegL src) %{
  5842     instruction_count(2);
  5843     dst    : S4(write);
  5844     src    : S3(read);
  5845     DECODE : S0(2);     // any 2 decoders
  5846     ALU    : S3(2);     // both alus
  5847 %}
  5849 // Integer ALU reg-reg operation
  5850 pipe_class ialu_reg_reg_fat(eRegI dst, memory src) %{
  5851     single_instruction;
  5852     dst    : S4(write);
  5853     src    : S3(read);
  5854     D0     : S0;        // big decoder only
  5855     ALU    : S3;        // any alu
  5856 %}
  5858 // Long ALU reg-reg operation
  5859 pipe_class ialu_reg_reg_long_fat(eRegL dst, eRegL src) %{
  5860     instruction_count(2);
  5861     dst    : S4(write);
  5862     src    : S3(read);
  5863     D0     : S0(2);     // big decoder only; twice
  5864     ALU    : S3(2);     // both alus
  5865 %}
  5867 // Integer ALU reg-mem operation
  5868 pipe_class ialu_reg_mem(eRegI dst, memory mem) %{
  5869     single_instruction;
  5870     dst    : S5(write);
  5871     mem    : S3(read);
  5872     D0     : S0;        // big decoder only
  5873     ALU    : S4;        // any alu
  5874     MEM    : S3;        // any mem
  5875 %}
  5877 // Long ALU reg-mem operation
  5878 pipe_class ialu_reg_long_mem(eRegL dst, load_long_memory mem) %{
  5879     instruction_count(2);
  5880     dst    : S5(write);
  5881     mem    : S3(read);
  5882     D0     : S0(2);     // big decoder only; twice
  5883     ALU    : S4(2);     // any 2 alus
  5884     MEM    : S3(2);     // both mems
  5885 %}
  5887 // Integer mem operation (prefetch)
  5888 pipe_class ialu_mem(memory mem)
  5889 %{
  5890     single_instruction;
  5891     mem    : S3(read);
  5892     D0     : S0;        // big decoder only
  5893     MEM    : S3;        // any mem
  5894 %}
  5896 // Integer Store to Memory
  5897 pipe_class ialu_mem_reg(memory mem, eRegI src) %{
  5898     single_instruction;
  5899     mem    : S3(read);
  5900     src    : S5(read);
  5901     D0     : S0;        // big decoder only
  5902     ALU    : S4;        // any alu
  5903     MEM    : S3;
  5904 %}
  5906 // Long Store to Memory
  5907 pipe_class ialu_mem_long_reg(memory mem, eRegL src) %{
  5908     instruction_count(2);
  5909     mem    : S3(read);
  5910     src    : S5(read);
  5911     D0     : S0(2);     // big decoder only; twice
  5912     ALU    : S4(2);     // any 2 alus
  5913     MEM    : S3(2);     // Both mems
  5914 %}
  5916 // Integer Store to Memory
  5917 pipe_class ialu_mem_imm(memory mem) %{
  5918     single_instruction;
  5919     mem    : S3(read);
  5920     D0     : S0;        // big decoder only
  5921     ALU    : S4;        // any alu
  5922     MEM    : S3;
  5923 %}
  5925 // Integer ALU0 reg-reg operation
  5926 pipe_class ialu_reg_reg_alu0(eRegI dst, eRegI src) %{
  5927     single_instruction;
  5928     dst    : S4(write);
  5929     src    : S3(read);
  5930     D0     : S0;        // Big decoder only
  5931     ALU0   : S3;        // only alu0
  5932 %}
  5934 // Integer ALU0 reg-mem operation
  5935 pipe_class ialu_reg_mem_alu0(eRegI dst, memory mem) %{
  5936     single_instruction;
  5937     dst    : S5(write);
  5938     mem    : S3(read);
  5939     D0     : S0;        // big decoder only
  5940     ALU0   : S4;        // ALU0 only
  5941     MEM    : S3;        // any mem
  5942 %}
  5944 // Integer ALU reg-reg operation
  5945 pipe_class ialu_cr_reg_reg(eFlagsReg cr, eRegI src1, eRegI src2) %{
  5946     single_instruction;
  5947     cr     : S4(write);
  5948     src1   : S3(read);
  5949     src2   : S3(read);
  5950     DECODE : S0;        // any decoder
  5951     ALU    : S3;        // any alu
  5952 %}
  5954 // Integer ALU reg-imm operation
  5955 pipe_class ialu_cr_reg_imm(eFlagsReg cr, eRegI src1) %{
  5956     single_instruction;
  5957     cr     : S4(write);
  5958     src1   : S3(read);
  5959     DECODE : S0;        // any decoder
  5960     ALU    : S3;        // any alu
  5961 %}
  5963 // Integer ALU reg-mem operation
  5964 pipe_class ialu_cr_reg_mem(eFlagsReg cr, eRegI src1, memory src2) %{
  5965     single_instruction;
  5966     cr     : S4(write);
  5967     src1   : S3(read);
  5968     src2   : S3(read);
  5969     D0     : S0;        // big decoder only
  5970     ALU    : S4;        // any alu
  5971     MEM    : S3;
  5972 %}
  5974 // Conditional move reg-reg
  5975 pipe_class pipe_cmplt( eRegI p, eRegI q, eRegI y ) %{
  5976     instruction_count(4);
  5977     y      : S4(read);
  5978     q      : S3(read);
  5979     p      : S3(read);
  5980     DECODE : S0(4);     // any decoder
  5981 %}
  5983 // Conditional move reg-reg
  5984 pipe_class pipe_cmov_reg( eRegI dst, eRegI src, eFlagsReg cr ) %{
  5985     single_instruction;
  5986     dst    : S4(write);
  5987     src    : S3(read);
  5988     cr     : S3(read);
  5989     DECODE : S0;        // any decoder
  5990 %}
  5992 // Conditional move reg-mem
  5993 pipe_class pipe_cmov_mem( eFlagsReg cr, eRegI dst, memory src) %{
  5994     single_instruction;
  5995     dst    : S4(write);
  5996     src    : S3(read);
  5997     cr     : S3(read);
  5998     DECODE : S0;        // any decoder
  5999     MEM    : S3;
  6000 %}
  6002 // Conditional move reg-reg long
  6003 pipe_class pipe_cmov_reg_long( eFlagsReg cr, eRegL dst, eRegL src) %{
  6004     single_instruction;
  6005     dst    : S4(write);
  6006     src    : S3(read);
  6007     cr     : S3(read);
  6008     DECODE : S0(2);     // any 2 decoders
  6009 %}
  6011 // Conditional move double reg-reg
  6012 pipe_class pipe_cmovD_reg( eFlagsReg cr, regDPR1 dst, regD src) %{
  6013     single_instruction;
  6014     dst    : S4(write);
  6015     src    : S3(read);
  6016     cr     : S3(read);
  6017     DECODE : S0;        // any decoder
  6018 %}
  6020 // Float reg-reg operation
  6021 pipe_class fpu_reg(regD dst) %{
  6022     instruction_count(2);
  6023     dst    : S3(read);
  6024     DECODE : S0(2);     // any 2 decoders
  6025     FPU    : S3;
  6026 %}
  6028 // Float reg-reg operation
  6029 pipe_class fpu_reg_reg(regD dst, regD src) %{
  6030     instruction_count(2);
  6031     dst    : S4(write);
  6032     src    : S3(read);
  6033     DECODE : S0(2);     // any 2 decoders
  6034     FPU    : S3;
  6035 %}
  6037 // Float reg-reg operation
  6038 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2) %{
  6039     instruction_count(3);
  6040     dst    : S4(write);
  6041     src1   : S3(read);
  6042     src2   : S3(read);
  6043     DECODE : S0(3);     // any 3 decoders
  6044     FPU    : S3(2);
  6045 %}
  6047 // Float reg-reg operation
  6048 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3) %{
  6049     instruction_count(4);
  6050     dst    : S4(write);
  6051     src1   : S3(read);
  6052     src2   : S3(read);
  6053     src3   : S3(read);
  6054     DECODE : S0(4);     // any 3 decoders
  6055     FPU    : S3(2);
  6056 %}
  6058 // Float reg-reg operation
  6059 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3) %{
  6060     instruction_count(4);
  6061     dst    : S4(write);
  6062     src1   : S3(read);
  6063     src2   : S3(read);
  6064     src3   : S3(read);
  6065     DECODE : S1(3);     // any 3 decoders
  6066     D0     : S0;        // Big decoder only
  6067     FPU    : S3(2);
  6068     MEM    : S3;
  6069 %}
  6071 // Float reg-mem operation
  6072 pipe_class fpu_reg_mem(regD dst, memory mem) %{
  6073     instruction_count(2);
  6074     dst    : S5(write);
  6075     mem    : S3(read);
  6076     D0     : S0;        // big decoder only
  6077     DECODE : S1;        // any decoder for FPU POP
  6078     FPU    : S4;
  6079     MEM    : S3;        // any mem
  6080 %}
  6082 // Float reg-mem operation
  6083 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem) %{
  6084     instruction_count(3);
  6085     dst    : S5(write);
  6086     src1   : S3(read);
  6087     mem    : S3(read);
  6088     D0     : S0;        // big decoder only
  6089     DECODE : S1(2);     // any decoder for FPU POP
  6090     FPU    : S4;
  6091     MEM    : S3;        // any mem
  6092 %}
  6094 // Float mem-reg operation
  6095 pipe_class fpu_mem_reg(memory mem, regD src) %{
  6096     instruction_count(2);
  6097     src    : S5(read);
  6098     mem    : S3(read);
  6099     DECODE : S0;        // any decoder for FPU PUSH
  6100     D0     : S1;        // big decoder only
  6101     FPU    : S4;
  6102     MEM    : S3;        // any mem
  6103 %}
  6105 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2) %{
  6106     instruction_count(3);
  6107     src1   : S3(read);
  6108     src2   : S3(read);
  6109     mem    : S3(read);
  6110     DECODE : S0(2);     // any decoder for FPU PUSH
  6111     D0     : S1;        // big decoder only
  6112     FPU    : S4;
  6113     MEM    : S3;        // any mem
  6114 %}
  6116 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2) %{
  6117     instruction_count(3);
  6118     src1   : S3(read);
  6119     src2   : S3(read);
  6120     mem    : S4(read);
  6121     DECODE : S0;        // any decoder for FPU PUSH
  6122     D0     : S0(2);     // big decoder only
  6123     FPU    : S4;
  6124     MEM    : S3(2);     // any mem
  6125 %}
  6127 pipe_class fpu_mem_mem(memory dst, memory src1) %{
  6128     instruction_count(2);
  6129     src1   : S3(read);
  6130     dst    : S4(read);
  6131     D0     : S0(2);     // big decoder only
  6132     MEM    : S3(2);     // any mem
  6133 %}
  6135 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2) %{
  6136     instruction_count(3);
  6137     src1   : S3(read);
  6138     src2   : S3(read);
  6139     dst    : S4(read);
  6140     D0     : S0(3);     // big decoder only
  6141     FPU    : S4;
  6142     MEM    : S3(3);     // any mem
  6143 %}
  6145 pipe_class fpu_mem_reg_con(memory mem, regD src1) %{
  6146     instruction_count(3);
  6147     src1   : S4(read);
  6148     mem    : S4(read);
  6149     DECODE : S0;        // any decoder for FPU PUSH
  6150     D0     : S0(2);     // big decoder only
  6151     FPU    : S4;
  6152     MEM    : S3(2);     // any mem
  6153 %}
  6155 // Float load constant
  6156 pipe_class fpu_reg_con(regD dst) %{
  6157     instruction_count(2);
  6158     dst    : S5(write);
  6159     D0     : S0;        // big decoder only for the load
  6160     DECODE : S1;        // any decoder for FPU POP
  6161     FPU    : S4;
  6162     MEM    : S3;        // any mem
  6163 %}
  6165 // Float load constant
  6166 pipe_class fpu_reg_reg_con(regD dst, regD src) %{
  6167     instruction_count(3);
  6168     dst    : S5(write);
  6169     src    : S3(read);
  6170     D0     : S0;        // big decoder only for the load
  6171     DECODE : S1(2);     // any decoder for FPU POP
  6172     FPU    : S4;
  6173     MEM    : S3;        // any mem
  6174 %}
  6176 // UnConditional branch
  6177 pipe_class pipe_jmp( label labl ) %{
  6178     single_instruction;
  6179     BR   : S3;
  6180 %}
  6182 // Conditional branch
  6183 pipe_class pipe_jcc( cmpOp cmp, eFlagsReg cr, label labl ) %{
  6184     single_instruction;
  6185     cr    : S1(read);
  6186     BR    : S3;
  6187 %}
  6189 // Allocation idiom
  6190 pipe_class pipe_cmpxchg( eRegP dst, eRegP heap_ptr ) %{
  6191     instruction_count(1); force_serialization;
  6192     fixed_latency(6);
  6193     heap_ptr : S3(read);
  6194     DECODE   : S0(3);
  6195     D0       : S2;
  6196     MEM      : S3;
  6197     ALU      : S3(2);
  6198     dst      : S5(write);
  6199     BR       : S5;
  6200 %}
  6202 // Generic big/slow expanded idiom
  6203 pipe_class pipe_slow(  ) %{
  6204     instruction_count(10); multiple_bundles; force_serialization;
  6205     fixed_latency(100);
  6206     D0  : S0(2);
  6207     MEM : S3(2);
  6208 %}
  6210 // The real do-nothing guy
  6211 pipe_class empty( ) %{
  6212     instruction_count(0);
  6213 %}
  6215 // Define the class for the Nop node
  6216 define %{
  6217    MachNop = empty;
  6218 %}
  6220 %}
  6222 //----------INSTRUCTIONS-------------------------------------------------------
  6223 //
  6224 // match      -- States which machine-independent subtree may be replaced
  6225 //               by this instruction.
  6226 // ins_cost   -- The estimated cost of this instruction is used by instruction
  6227 //               selection to identify a minimum cost tree of machine
  6228 //               instructions that matches a tree of machine-independent
  6229 //               instructions.
  6230 // format     -- A string providing the disassembly for this instruction.
  6231 //               The value of an instruction's operand may be inserted
  6232 //               by referring to it with a '$' prefix.
  6233 // opcode     -- Three instruction opcodes may be provided.  These are referred
  6234 //               to within an encode class as $primary, $secondary, and $tertiary
  6235 //               respectively.  The primary opcode is commonly used to
  6236 //               indicate the type of machine instruction, while secondary
  6237 //               and tertiary are often used for prefix options or addressing
  6238 //               modes.
  6239 // ins_encode -- A list of encode classes with parameters. The encode class
  6240 //               name must have been defined in an 'enc_class' specification
  6241 //               in the encode section of the architecture description.
  6243 //----------BSWAP-Instruction--------------------------------------------------
  6244 instruct bytes_reverse_int(eRegI dst) %{
  6245   match(Set dst (ReverseBytesI dst));
  6247   format %{ "BSWAP  $dst" %}
  6248   opcode(0x0F, 0xC8);
  6249   ins_encode( OpcP, OpcSReg(dst) );
  6250   ins_pipe( ialu_reg );
  6251 %}
  6253 instruct bytes_reverse_long(eRegL dst) %{
  6254   match(Set dst (ReverseBytesL dst));
  6256   format %{ "BSWAP  $dst.lo\n\t"
  6257             "BSWAP  $dst.hi\n\t"
  6258             "XCHG   $dst.lo $dst.hi" %}
  6260   ins_cost(125);
  6261   ins_encode( bswap_long_bytes(dst) );
  6262   ins_pipe( ialu_reg_reg);
  6263 %}
  6265 instruct bytes_reverse_unsigned_short(eRegI dst) %{
  6266   match(Set dst (ReverseBytesUS dst));
  6268   format %{ "BSWAP  $dst\n\t" 
  6269             "SHR    $dst,16\n\t" %}
  6270   ins_encode %{
  6271     __ bswapl($dst$$Register);
  6272     __ shrl($dst$$Register, 16); 
  6273   %}
  6274   ins_pipe( ialu_reg );
  6275 %}
  6277 instruct bytes_reverse_short(eRegI dst) %{
  6278   match(Set dst (ReverseBytesS dst));
  6280   format %{ "BSWAP  $dst\n\t" 
  6281             "SAR    $dst,16\n\t" %}
  6282   ins_encode %{
  6283     __ bswapl($dst$$Register);
  6284     __ sarl($dst$$Register, 16); 
  6285   %}
  6286   ins_pipe( ialu_reg );
  6287 %}
  6290 //---------- Zeros Count Instructions ------------------------------------------
  6292 instruct countLeadingZerosI(eRegI dst, eRegI src, eFlagsReg cr) %{
  6293   predicate(UseCountLeadingZerosInstruction);
  6294   match(Set dst (CountLeadingZerosI src));
  6295   effect(KILL cr);
  6297   format %{ "LZCNT  $dst, $src\t# count leading zeros (int)" %}
  6298   ins_encode %{
  6299     __ lzcntl($dst$$Register, $src$$Register);
  6300   %}
  6301   ins_pipe(ialu_reg);
  6302 %}
  6304 instruct countLeadingZerosI_bsr(eRegI dst, eRegI src, eFlagsReg cr) %{
  6305   predicate(!UseCountLeadingZerosInstruction);
  6306   match(Set dst (CountLeadingZerosI src));
  6307   effect(KILL cr);
  6309   format %{ "BSR    $dst, $src\t# count leading zeros (int)\n\t"
  6310             "JNZ    skip\n\t"
  6311             "MOV    $dst, -1\n"
  6312       "skip:\n\t"
  6313             "NEG    $dst\n\t"
  6314             "ADD    $dst, 31" %}
  6315   ins_encode %{
  6316     Register Rdst = $dst$$Register;
  6317     Register Rsrc = $src$$Register;
  6318     Label skip;
  6319     __ bsrl(Rdst, Rsrc);
  6320     __ jccb(Assembler::notZero, skip);
  6321     __ movl(Rdst, -1);
  6322     __ bind(skip);
  6323     __ negl(Rdst);
  6324     __ addl(Rdst, BitsPerInt - 1);
  6325   %}
  6326   ins_pipe(ialu_reg);
  6327 %}
  6329 instruct countLeadingZerosL(eRegI dst, eRegL src, eFlagsReg cr) %{
  6330   predicate(UseCountLeadingZerosInstruction);
  6331   match(Set dst (CountLeadingZerosL src));
  6332   effect(TEMP dst, KILL cr);
  6334   format %{ "LZCNT  $dst, $src.hi\t# count leading zeros (long)\n\t"
  6335             "JNC    done\n\t"
  6336             "LZCNT  $dst, $src.lo\n\t"
  6337             "ADD    $dst, 32\n"
  6338       "done:" %}
  6339   ins_encode %{
  6340     Register Rdst = $dst$$Register;
  6341     Register Rsrc = $src$$Register;
  6342     Label done;
  6343     __ lzcntl(Rdst, HIGH_FROM_LOW(Rsrc));
  6344     __ jccb(Assembler::carryClear, done);
  6345     __ lzcntl(Rdst, Rsrc);
  6346     __ addl(Rdst, BitsPerInt);
  6347     __ bind(done);
  6348   %}
  6349   ins_pipe(ialu_reg);
  6350 %}
  6352 instruct countLeadingZerosL_bsr(eRegI dst, eRegL src, eFlagsReg cr) %{
  6353   predicate(!UseCountLeadingZerosInstruction);
  6354   match(Set dst (CountLeadingZerosL src));
  6355   effect(TEMP dst, KILL cr);
  6357   format %{ "BSR    $dst, $src.hi\t# count leading zeros (long)\n\t"
  6358             "JZ     msw_is_zero\n\t"
  6359             "ADD    $dst, 32\n\t"
  6360             "JMP    not_zero\n"
  6361       "msw_is_zero:\n\t"
  6362             "BSR    $dst, $src.lo\n\t"
  6363             "JNZ    not_zero\n\t"
  6364             "MOV    $dst, -1\n"
  6365       "not_zero:\n\t"
  6366             "NEG    $dst\n\t"
  6367             "ADD    $dst, 63\n" %}
  6368  ins_encode %{
  6369     Register Rdst = $dst$$Register;
  6370     Register Rsrc = $src$$Register;
  6371     Label msw_is_zero;
  6372     Label not_zero;
  6373     __ bsrl(Rdst, HIGH_FROM_LOW(Rsrc));
  6374     __ jccb(Assembler::zero, msw_is_zero);
  6375     __ addl(Rdst, BitsPerInt);
  6376     __ jmpb(not_zero);
  6377     __ bind(msw_is_zero);
  6378     __ bsrl(Rdst, Rsrc);
  6379     __ jccb(Assembler::notZero, not_zero);
  6380     __ movl(Rdst, -1);
  6381     __ bind(not_zero);
  6382     __ negl(Rdst);
  6383     __ addl(Rdst, BitsPerLong - 1);
  6384   %}
  6385   ins_pipe(ialu_reg);
  6386 %}
  6388 instruct countTrailingZerosI(eRegI dst, eRegI src, eFlagsReg cr) %{
  6389   match(Set dst (CountTrailingZerosI src));
  6390   effect(KILL cr);
  6392   format %{ "BSF    $dst, $src\t# count trailing zeros (int)\n\t"
  6393             "JNZ    done\n\t"
  6394             "MOV    $dst, 32\n"
  6395       "done:" %}
  6396   ins_encode %{
  6397     Register Rdst = $dst$$Register;
  6398     Label done;
  6399     __ bsfl(Rdst, $src$$Register);
  6400     __ jccb(Assembler::notZero, done);
  6401     __ movl(Rdst, BitsPerInt);
  6402     __ bind(done);
  6403   %}
  6404   ins_pipe(ialu_reg);
  6405 %}
  6407 instruct countTrailingZerosL(eRegI dst, eRegL src, eFlagsReg cr) %{
  6408   match(Set dst (CountTrailingZerosL src));
  6409   effect(TEMP dst, KILL cr);
  6411   format %{ "BSF    $dst, $src.lo\t# count trailing zeros (long)\n\t"
  6412             "JNZ    done\n\t"
  6413             "BSF    $dst, $src.hi\n\t"
  6414             "JNZ    msw_not_zero\n\t"
  6415             "MOV    $dst, 32\n"
  6416       "msw_not_zero:\n\t"
  6417             "ADD    $dst, 32\n"
  6418       "done:" %}
  6419   ins_encode %{
  6420     Register Rdst = $dst$$Register;
  6421     Register Rsrc = $src$$Register;
  6422     Label msw_not_zero;
  6423     Label done;
  6424     __ bsfl(Rdst, Rsrc);
  6425     __ jccb(Assembler::notZero, done);
  6426     __ bsfl(Rdst, HIGH_FROM_LOW(Rsrc));
  6427     __ jccb(Assembler::notZero, msw_not_zero);
  6428     __ movl(Rdst, BitsPerInt);
  6429     __ bind(msw_not_zero);
  6430     __ addl(Rdst, BitsPerInt);
  6431     __ bind(done);
  6432   %}
  6433   ins_pipe(ialu_reg);
  6434 %}
  6437 //---------- Population Count Instructions -------------------------------------
  6439 instruct popCountI(eRegI dst, eRegI src) %{
  6440   predicate(UsePopCountInstruction);
  6441   match(Set dst (PopCountI src));
  6443   format %{ "POPCNT $dst, $src" %}
  6444   ins_encode %{
  6445     __ popcntl($dst$$Register, $src$$Register);
  6446   %}
  6447   ins_pipe(ialu_reg);
  6448 %}
  6450 instruct popCountI_mem(eRegI dst, memory mem) %{
  6451   predicate(UsePopCountInstruction);
  6452   match(Set dst (PopCountI (LoadI mem)));
  6454   format %{ "POPCNT $dst, $mem" %}
  6455   ins_encode %{
  6456     __ popcntl($dst$$Register, $mem$$Address);
  6457   %}
  6458   ins_pipe(ialu_reg);
  6459 %}
  6461 // Note: Long.bitCount(long) returns an int.
  6462 instruct popCountL(eRegI dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
  6463   predicate(UsePopCountInstruction);
  6464   match(Set dst (PopCountL src));
  6465   effect(KILL cr, TEMP tmp, TEMP dst);
  6467   format %{ "POPCNT $dst, $src.lo\n\t"
  6468             "POPCNT $tmp, $src.hi\n\t"
  6469             "ADD    $dst, $tmp" %}
  6470   ins_encode %{
  6471     __ popcntl($dst$$Register, $src$$Register);
  6472     __ popcntl($tmp$$Register, HIGH_FROM_LOW($src$$Register));
  6473     __ addl($dst$$Register, $tmp$$Register);
  6474   %}
  6475   ins_pipe(ialu_reg);
  6476 %}
  6478 // Note: Long.bitCount(long) returns an int.
  6479 instruct popCountL_mem(eRegI dst, memory mem, eRegI tmp, eFlagsReg cr) %{
  6480   predicate(UsePopCountInstruction);
  6481   match(Set dst (PopCountL (LoadL mem)));
  6482   effect(KILL cr, TEMP tmp, TEMP dst);
  6484   format %{ "POPCNT $dst, $mem\n\t"
  6485             "POPCNT $tmp, $mem+4\n\t"
  6486             "ADD    $dst, $tmp" %}
  6487   ins_encode %{
  6488     //__ popcntl($dst$$Register, $mem$$Address$$first);
  6489     //__ popcntl($tmp$$Register, $mem$$Address$$second);
  6490     __ popcntl($dst$$Register, Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, false));
  6491     __ popcntl($tmp$$Register, Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, false));
  6492     __ addl($dst$$Register, $tmp$$Register);
  6493   %}
  6494   ins_pipe(ialu_reg);
  6495 %}
  6498 //----------Load/Store/Move Instructions---------------------------------------
  6499 //----------Load Instructions--------------------------------------------------
  6500 // Load Byte (8bit signed)
  6501 instruct loadB(xRegI dst, memory mem) %{
  6502   match(Set dst (LoadB mem));
  6504   ins_cost(125);
  6505   format %{ "MOVSX8 $dst,$mem\t# byte" %}
  6507   ins_encode %{
  6508     __ movsbl($dst$$Register, $mem$$Address);
  6509   %}
  6511   ins_pipe(ialu_reg_mem);
  6512 %}
  6514 // Load Byte (8bit signed) into Long Register
  6515 instruct loadB2L(eRegL dst, memory mem, eFlagsReg cr) %{
  6516   match(Set dst (ConvI2L (LoadB mem)));
  6517   effect(KILL cr);
  6519   ins_cost(375);
  6520   format %{ "MOVSX8 $dst.lo,$mem\t# byte -> long\n\t"
  6521             "MOV    $dst.hi,$dst.lo\n\t"
  6522             "SAR    $dst.hi,7" %}
  6524   ins_encode %{
  6525     __ movsbl($dst$$Register, $mem$$Address);
  6526     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
  6527     __ sarl(HIGH_FROM_LOW($dst$$Register), 7); // 24+1 MSB are already signed extended.
  6528   %}
  6530   ins_pipe(ialu_reg_mem);
  6531 %}
  6533 // Load Unsigned Byte (8bit UNsigned)
  6534 instruct loadUB(xRegI dst, memory mem) %{
  6535   match(Set dst (LoadUB mem));
  6537   ins_cost(125);
  6538   format %{ "MOVZX8 $dst,$mem\t# ubyte -> int" %}
  6540   ins_encode %{
  6541     __ movzbl($dst$$Register, $mem$$Address);
  6542   %}
  6544   ins_pipe(ialu_reg_mem);
  6545 %}
  6547 // Load Unsigned Byte (8 bit UNsigned) into Long Register
  6548 instruct loadUB2L(eRegL dst, memory mem, eFlagsReg cr) %{
  6549   match(Set dst (ConvI2L (LoadUB mem)));
  6550   effect(KILL cr);
  6552   ins_cost(250);
  6553   format %{ "MOVZX8 $dst.lo,$mem\t# ubyte -> long\n\t"
  6554             "XOR    $dst.hi,$dst.hi" %}
  6556   ins_encode %{
  6557     Register Rdst = $dst$$Register;
  6558     __ movzbl(Rdst, $mem$$Address);
  6559     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  6560   %}
  6562   ins_pipe(ialu_reg_mem);
  6563 %}
  6565 // Load Unsigned Byte (8 bit UNsigned) with mask into Long Register
  6566 instruct loadUB2L_immI8(eRegL dst, memory mem, immI8 mask, eFlagsReg cr) %{
  6567   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
  6568   effect(KILL cr);
  6570   format %{ "MOVZX8 $dst.lo,$mem\t# ubyte & 8-bit mask -> long\n\t"
  6571             "XOR    $dst.hi,$dst.hi\n\t"
  6572             "AND    $dst.lo,$mask" %}
  6573   ins_encode %{
  6574     Register Rdst = $dst$$Register;
  6575     __ movzbl(Rdst, $mem$$Address);
  6576     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  6577     __ andl(Rdst, $mask$$constant);
  6578   %}
  6579   ins_pipe(ialu_reg_mem);
  6580 %}
  6582 // Load Short (16bit signed)
  6583 instruct loadS(eRegI dst, memory mem) %{
  6584   match(Set dst (LoadS mem));
  6586   ins_cost(125);
  6587   format %{ "MOVSX  $dst,$mem\t# short" %}
  6589   ins_encode %{
  6590     __ movswl($dst$$Register, $mem$$Address);
  6591   %}
  6593   ins_pipe(ialu_reg_mem);
  6594 %}
  6596 // Load Short (16 bit signed) to Byte (8 bit signed)
  6597 instruct loadS2B(eRegI dst, memory mem, immI_24 twentyfour) %{
  6598   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
  6600   ins_cost(125);
  6601   format %{ "MOVSX  $dst, $mem\t# short -> byte" %}
  6602   ins_encode %{
  6603     __ movsbl($dst$$Register, $mem$$Address);
  6604   %}
  6605   ins_pipe(ialu_reg_mem);
  6606 %}
  6608 // Load Short (16bit signed) into Long Register
  6609 instruct loadS2L(eRegL dst, memory mem, eFlagsReg cr) %{
  6610   match(Set dst (ConvI2L (LoadS mem)));
  6611   effect(KILL cr);
  6613   ins_cost(375);
  6614   format %{ "MOVSX  $dst.lo,$mem\t# short -> long\n\t"
  6615             "MOV    $dst.hi,$dst.lo\n\t"
  6616             "SAR    $dst.hi,15" %}
  6618   ins_encode %{
  6619     __ movswl($dst$$Register, $mem$$Address);
  6620     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
  6621     __ sarl(HIGH_FROM_LOW($dst$$Register), 15); // 16+1 MSB are already signed extended.
  6622   %}
  6624   ins_pipe(ialu_reg_mem);
  6625 %}
  6627 // Load Unsigned Short/Char (16bit unsigned)
  6628 instruct loadUS(eRegI dst, memory mem) %{
  6629   match(Set dst (LoadUS mem));
  6631   ins_cost(125);
  6632   format %{ "MOVZX  $dst,$mem\t# ushort/char -> int" %}
  6634   ins_encode %{
  6635     __ movzwl($dst$$Register, $mem$$Address);
  6636   %}
  6638   ins_pipe(ialu_reg_mem);
  6639 %}
  6641 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
  6642 instruct loadUS2B(eRegI dst, memory mem, immI_24 twentyfour) %{
  6643   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
  6645   ins_cost(125);
  6646   format %{ "MOVSX  $dst, $mem\t# ushort -> byte" %}
  6647   ins_encode %{
  6648     __ movsbl($dst$$Register, $mem$$Address);
  6649   %}
  6650   ins_pipe(ialu_reg_mem);
  6651 %}
  6653 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
  6654 instruct loadUS2L(eRegL dst, memory mem, eFlagsReg cr) %{
  6655   match(Set dst (ConvI2L (LoadUS mem)));
  6656   effect(KILL cr);
  6658   ins_cost(250);
  6659   format %{ "MOVZX  $dst.lo,$mem\t# ushort/char -> long\n\t"
  6660             "XOR    $dst.hi,$dst.hi" %}
  6662   ins_encode %{
  6663     __ movzwl($dst$$Register, $mem$$Address);
  6664     __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
  6665   %}
  6667   ins_pipe(ialu_reg_mem);
  6668 %}
  6670 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
  6671 instruct loadUS2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
  6672   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
  6673   effect(KILL cr);
  6675   format %{ "MOVZX8 $dst.lo,$mem\t# ushort/char & 0xFF -> long\n\t"
  6676             "XOR    $dst.hi,$dst.hi" %}
  6677   ins_encode %{
  6678     Register Rdst = $dst$$Register;
  6679     __ movzbl(Rdst, $mem$$Address);
  6680     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  6681   %}
  6682   ins_pipe(ialu_reg_mem);
  6683 %}
  6685 // Load Unsigned Short/Char (16 bit UNsigned) with a 16-bit mask into Long Register
  6686 instruct loadUS2L_immI16(eRegL dst, memory mem, immI16 mask, eFlagsReg cr) %{
  6687   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
  6688   effect(KILL cr);
  6690   format %{ "MOVZX  $dst.lo, $mem\t# ushort/char & 16-bit mask -> long\n\t"
  6691             "XOR    $dst.hi,$dst.hi\n\t"
  6692             "AND    $dst.lo,$mask" %}
  6693   ins_encode %{
  6694     Register Rdst = $dst$$Register;
  6695     __ movzwl(Rdst, $mem$$Address);
  6696     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  6697     __ andl(Rdst, $mask$$constant);
  6698   %}
  6699   ins_pipe(ialu_reg_mem);
  6700 %}
  6702 // Load Integer
  6703 instruct loadI(eRegI dst, memory mem) %{
  6704   match(Set dst (LoadI mem));
  6706   ins_cost(125);
  6707   format %{ "MOV    $dst,$mem\t# int" %}
  6709   ins_encode %{
  6710     __ movl($dst$$Register, $mem$$Address);
  6711   %}
  6713   ins_pipe(ialu_reg_mem);
  6714 %}
  6716 // Load Integer (32 bit signed) to Byte (8 bit signed)
  6717 instruct loadI2B(eRegI dst, memory mem, immI_24 twentyfour) %{
  6718   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
  6720   ins_cost(125);
  6721   format %{ "MOVSX  $dst, $mem\t# int -> byte" %}
  6722   ins_encode %{
  6723     __ movsbl($dst$$Register, $mem$$Address);
  6724   %}
  6725   ins_pipe(ialu_reg_mem);
  6726 %}
  6728 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
  6729 instruct loadI2UB(eRegI dst, memory mem, immI_255 mask) %{
  6730   match(Set dst (AndI (LoadI mem) mask));
  6732   ins_cost(125);
  6733   format %{ "MOVZX  $dst, $mem\t# int -> ubyte" %}
  6734   ins_encode %{
  6735     __ movzbl($dst$$Register, $mem$$Address);
  6736   %}
  6737   ins_pipe(ialu_reg_mem);
  6738 %}
  6740 // Load Integer (32 bit signed) to Short (16 bit signed)
  6741 instruct loadI2S(eRegI dst, memory mem, immI_16 sixteen) %{
  6742   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
  6744   ins_cost(125);
  6745   format %{ "MOVSX  $dst, $mem\t# int -> short" %}
  6746   ins_encode %{
  6747     __ movswl($dst$$Register, $mem$$Address);
  6748   %}
  6749   ins_pipe(ialu_reg_mem);
  6750 %}
  6752 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
  6753 instruct loadI2US(eRegI dst, memory mem, immI_65535 mask) %{
  6754   match(Set dst (AndI (LoadI mem) mask));
  6756   ins_cost(125);
  6757   format %{ "MOVZX  $dst, $mem\t# int -> ushort/char" %}
  6758   ins_encode %{
  6759     __ movzwl($dst$$Register, $mem$$Address);
  6760   %}
  6761   ins_pipe(ialu_reg_mem);
  6762 %}
  6764 // Load Integer into Long Register
  6765 instruct loadI2L(eRegL dst, memory mem, eFlagsReg cr) %{
  6766   match(Set dst (ConvI2L (LoadI mem)));
  6767   effect(KILL cr);
  6769   ins_cost(375);
  6770   format %{ "MOV    $dst.lo,$mem\t# int -> long\n\t"
  6771             "MOV    $dst.hi,$dst.lo\n\t"
  6772             "SAR    $dst.hi,31" %}
  6774   ins_encode %{
  6775     __ movl($dst$$Register, $mem$$Address);
  6776     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
  6777     __ sarl(HIGH_FROM_LOW($dst$$Register), 31);
  6778   %}
  6780   ins_pipe(ialu_reg_mem);
  6781 %}
  6783 // Load Integer with mask 0xFF into Long Register
  6784 instruct loadI2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
  6785   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  6786   effect(KILL cr);
  6788   format %{ "MOVZX8 $dst.lo,$mem\t# int & 0xFF -> long\n\t"
  6789             "XOR    $dst.hi,$dst.hi" %}
  6790   ins_encode %{
  6791     Register Rdst = $dst$$Register;
  6792     __ movzbl(Rdst, $mem$$Address);
  6793     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  6794   %}
  6795   ins_pipe(ialu_reg_mem);
  6796 %}
  6798 // Load Integer with mask 0xFFFF into Long Register
  6799 instruct loadI2L_immI_65535(eRegL dst, memory mem, immI_65535 mask, eFlagsReg cr) %{
  6800   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  6801   effect(KILL cr);
  6803   format %{ "MOVZX  $dst.lo,$mem\t# int & 0xFFFF -> long\n\t"
  6804             "XOR    $dst.hi,$dst.hi" %}
  6805   ins_encode %{
  6806     Register Rdst = $dst$$Register;
  6807     __ movzwl(Rdst, $mem$$Address);
  6808     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  6809   %}
  6810   ins_pipe(ialu_reg_mem);
  6811 %}
  6813 // Load Integer with 32-bit mask into Long Register
  6814 instruct loadI2L_immI(eRegL dst, memory mem, immI mask, eFlagsReg cr) %{
  6815   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
  6816   effect(KILL cr);
  6818   format %{ "MOV    $dst.lo,$mem\t# int & 32-bit mask -> long\n\t"
  6819             "XOR    $dst.hi,$dst.hi\n\t"
  6820             "AND    $dst.lo,$mask" %}
  6821   ins_encode %{
  6822     Register Rdst = $dst$$Register;
  6823     __ movl(Rdst, $mem$$Address);
  6824     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
  6825     __ andl(Rdst, $mask$$constant);
  6826   %}
  6827   ins_pipe(ialu_reg_mem);
  6828 %}
  6830 // Load Unsigned Integer into Long Register
  6831 instruct loadUI2L(eRegL dst, memory mem, eFlagsReg cr) %{
  6832   match(Set dst (LoadUI2L mem));
  6833   effect(KILL cr);
  6835   ins_cost(250);
  6836   format %{ "MOV    $dst.lo,$mem\t# uint -> long\n\t"
  6837             "XOR    $dst.hi,$dst.hi" %}
  6839   ins_encode %{
  6840     __ movl($dst$$Register, $mem$$Address);
  6841     __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
  6842   %}
  6844   ins_pipe(ialu_reg_mem);
  6845 %}
  6847 // Load Long.  Cannot clobber address while loading, so restrict address
  6848 // register to ESI
  6849 instruct loadL(eRegL dst, load_long_memory mem) %{
  6850   predicate(!((LoadLNode*)n)->require_atomic_access());
  6851   match(Set dst (LoadL mem));
  6853   ins_cost(250);
  6854   format %{ "MOV    $dst.lo,$mem\t# long\n\t"
  6855             "MOV    $dst.hi,$mem+4" %}
  6857   ins_encode %{
  6858     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, false);
  6859     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, false);
  6860     __ movl($dst$$Register, Amemlo);
  6861     __ movl(HIGH_FROM_LOW($dst$$Register), Amemhi);
  6862   %}
  6864   ins_pipe(ialu_reg_long_mem);
  6865 %}
  6867 // Volatile Load Long.  Must be atomic, so do 64-bit FILD
  6868 // then store it down to the stack and reload on the int
  6869 // side.
  6870 instruct loadL_volatile(stackSlotL dst, memory mem) %{
  6871   predicate(UseSSE<=1 && ((LoadLNode*)n)->require_atomic_access());
  6872   match(Set dst (LoadL mem));
  6874   ins_cost(200);
  6875   format %{ "FILD   $mem\t# Atomic volatile long load\n\t"
  6876             "FISTp  $dst" %}
  6877   ins_encode(enc_loadL_volatile(mem,dst));
  6878   ins_pipe( fpu_reg_mem );
  6879 %}
  6881 instruct loadLX_volatile(stackSlotL dst, memory mem, regXD tmp) %{
  6882   predicate(UseSSE>=2 && ((LoadLNode*)n)->require_atomic_access());
  6883   match(Set dst (LoadL mem));
  6884   effect(TEMP tmp);
  6885   ins_cost(180);
  6886   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
  6887             "MOVSD  $dst,$tmp" %}
  6888   ins_encode(enc_loadLX_volatile(mem, dst, tmp));
  6889   ins_pipe( pipe_slow );
  6890 %}
  6892 instruct loadLX_reg_volatile(eRegL dst, memory mem, regXD tmp) %{
  6893   predicate(UseSSE>=2 && ((LoadLNode*)n)->require_atomic_access());
  6894   match(Set dst (LoadL mem));
  6895   effect(TEMP tmp);
  6896   ins_cost(160);
  6897   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
  6898             "MOVD   $dst.lo,$tmp\n\t"
  6899             "PSRLQ  $tmp,32\n\t"
  6900             "MOVD   $dst.hi,$tmp" %}
  6901   ins_encode(enc_loadLX_reg_volatile(mem, dst, tmp));
  6902   ins_pipe( pipe_slow );
  6903 %}
  6905 // Load Range
  6906 instruct loadRange(eRegI dst, memory mem) %{
  6907   match(Set dst (LoadRange mem));
  6909   ins_cost(125);
  6910   format %{ "MOV    $dst,$mem" %}
  6911   opcode(0x8B);
  6912   ins_encode( OpcP, RegMem(dst,mem));
  6913   ins_pipe( ialu_reg_mem );
  6914 %}
  6917 // Load Pointer
  6918 instruct loadP(eRegP dst, memory mem) %{
  6919   match(Set dst (LoadP mem));
  6921   ins_cost(125);
  6922   format %{ "MOV    $dst,$mem" %}
  6923   opcode(0x8B);
  6924   ins_encode( OpcP, RegMem(dst,mem));
  6925   ins_pipe( ialu_reg_mem );
  6926 %}
  6928 // Load Klass Pointer
  6929 instruct loadKlass(eRegP dst, memory mem) %{
  6930   match(Set dst (LoadKlass mem));
  6932   ins_cost(125);
  6933   format %{ "MOV    $dst,$mem" %}
  6934   opcode(0x8B);
  6935   ins_encode( OpcP, RegMem(dst,mem));
  6936   ins_pipe( ialu_reg_mem );
  6937 %}
  6939 // Load Double
  6940 instruct loadD(regD dst, memory mem) %{
  6941   predicate(UseSSE<=1);
  6942   match(Set dst (LoadD mem));
  6944   ins_cost(150);
  6945   format %{ "FLD_D  ST,$mem\n\t"
  6946             "FSTP   $dst" %}
  6947   opcode(0xDD);               /* DD /0 */
  6948   ins_encode( OpcP, RMopc_Mem(0x00,mem),
  6949               Pop_Reg_D(dst) );
  6950   ins_pipe( fpu_reg_mem );
  6951 %}
  6953 // Load Double to XMM
  6954 instruct loadXD(regXD dst, memory mem) %{
  6955   predicate(UseSSE>=2 && UseXmmLoadAndClearUpper);
  6956   match(Set dst (LoadD mem));
  6957   ins_cost(145);
  6958   format %{ "MOVSD  $dst,$mem" %}
  6959   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x10), RegMem(dst,mem));
  6960   ins_pipe( pipe_slow );
  6961 %}
  6963 instruct loadXD_partial(regXD dst, memory mem) %{
  6964   predicate(UseSSE>=2 && !UseXmmLoadAndClearUpper);
  6965   match(Set dst (LoadD mem));
  6966   ins_cost(145);
  6967   format %{ "MOVLPD $dst,$mem" %}
  6968   ins_encode( Opcode(0x66), Opcode(0x0F), Opcode(0x12), RegMem(dst,mem));
  6969   ins_pipe( pipe_slow );
  6970 %}
  6972 // Load to XMM register (single-precision floating point)
  6973 // MOVSS instruction
  6974 instruct loadX(regX dst, memory mem) %{
  6975   predicate(UseSSE>=1);
  6976   match(Set dst (LoadF mem));
  6977   ins_cost(145);
  6978   format %{ "MOVSS  $dst,$mem" %}
  6979   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x10), RegMem(dst,mem));
  6980   ins_pipe( pipe_slow );
  6981 %}
  6983 // Load Float
  6984 instruct loadF(regF dst, memory mem) %{
  6985   predicate(UseSSE==0);
  6986   match(Set dst (LoadF mem));
  6988   ins_cost(150);
  6989   format %{ "FLD_S  ST,$mem\n\t"
  6990             "FSTP   $dst" %}
  6991   opcode(0xD9);               /* D9 /0 */
  6992   ins_encode( OpcP, RMopc_Mem(0x00,mem),
  6993               Pop_Reg_F(dst) );
  6994   ins_pipe( fpu_reg_mem );
  6995 %}
  6997 // Load Aligned Packed Byte to XMM register
  6998 instruct loadA8B(regXD dst, memory mem) %{
  6999   predicate(UseSSE>=1);
  7000   match(Set dst (Load8B mem));
  7001   ins_cost(125);
  7002   format %{ "MOVQ  $dst,$mem\t! packed8B" %}
  7003   ins_encode( movq_ld(dst, mem));
  7004   ins_pipe( pipe_slow );
  7005 %}
  7007 // Load Aligned Packed Short to XMM register
  7008 instruct loadA4S(regXD dst, memory mem) %{
  7009   predicate(UseSSE>=1);
  7010   match(Set dst (Load4S mem));
  7011   ins_cost(125);
  7012   format %{ "MOVQ  $dst,$mem\t! packed4S" %}
  7013   ins_encode( movq_ld(dst, mem));
  7014   ins_pipe( pipe_slow );
  7015 %}
  7017 // Load Aligned Packed Char to XMM register
  7018 instruct loadA4C(regXD dst, memory mem) %{
  7019   predicate(UseSSE>=1);
  7020   match(Set dst (Load4C mem));
  7021   ins_cost(125);
  7022   format %{ "MOVQ  $dst,$mem\t! packed4C" %}
  7023   ins_encode( movq_ld(dst, mem));
  7024   ins_pipe( pipe_slow );
  7025 %}
  7027 // Load Aligned Packed Integer to XMM register
  7028 instruct load2IU(regXD dst, memory mem) %{
  7029   predicate(UseSSE>=1);
  7030   match(Set dst (Load2I mem));
  7031   ins_cost(125);
  7032   format %{ "MOVQ  $dst,$mem\t! packed2I" %}
  7033   ins_encode( movq_ld(dst, mem));
  7034   ins_pipe( pipe_slow );
  7035 %}
  7037 // Load Aligned Packed Single to XMM
  7038 instruct loadA2F(regXD dst, memory mem) %{
  7039   predicate(UseSSE>=1);
  7040   match(Set dst (Load2F mem));
  7041   ins_cost(145);
  7042   format %{ "MOVQ  $dst,$mem\t! packed2F" %}
  7043   ins_encode( movq_ld(dst, mem));
  7044   ins_pipe( pipe_slow );
  7045 %}
  7047 // Load Effective Address
  7048 instruct leaP8(eRegP dst, indOffset8 mem) %{
  7049   match(Set dst mem);
  7051   ins_cost(110);
  7052   format %{ "LEA    $dst,$mem" %}
  7053   opcode(0x8D);
  7054   ins_encode( OpcP, RegMem(dst,mem));
  7055   ins_pipe( ialu_reg_reg_fat );
  7056 %}
  7058 instruct leaP32(eRegP dst, indOffset32 mem) %{
  7059   match(Set dst mem);
  7061   ins_cost(110);
  7062   format %{ "LEA    $dst,$mem" %}
  7063   opcode(0x8D);
  7064   ins_encode( OpcP, RegMem(dst,mem));
  7065   ins_pipe( ialu_reg_reg_fat );
  7066 %}
  7068 instruct leaPIdxOff(eRegP dst, indIndexOffset mem) %{
  7069   match(Set dst mem);
  7071   ins_cost(110);
  7072   format %{ "LEA    $dst,$mem" %}
  7073   opcode(0x8D);
  7074   ins_encode( OpcP, RegMem(dst,mem));
  7075   ins_pipe( ialu_reg_reg_fat );
  7076 %}
  7078 instruct leaPIdxScale(eRegP dst, indIndexScale mem) %{
  7079   match(Set dst mem);
  7081   ins_cost(110);
  7082   format %{ "LEA    $dst,$mem" %}
  7083   opcode(0x8D);
  7084   ins_encode( OpcP, RegMem(dst,mem));
  7085   ins_pipe( ialu_reg_reg_fat );
  7086 %}
  7088 instruct leaPIdxScaleOff(eRegP dst, indIndexScaleOffset mem) %{
  7089   match(Set dst mem);
  7091   ins_cost(110);
  7092   format %{ "LEA    $dst,$mem" %}
  7093   opcode(0x8D);
  7094   ins_encode( OpcP, RegMem(dst,mem));
  7095   ins_pipe( ialu_reg_reg_fat );
  7096 %}
  7098 // Load Constant
  7099 instruct loadConI(eRegI dst, immI src) %{
  7100   match(Set dst src);
  7102   format %{ "MOV    $dst,$src" %}
  7103   ins_encode( LdImmI(dst, src) );
  7104   ins_pipe( ialu_reg_fat );
  7105 %}
  7107 // Load Constant zero
  7108 instruct loadConI0(eRegI dst, immI0 src, eFlagsReg cr) %{
  7109   match(Set dst src);
  7110   effect(KILL cr);
  7112   ins_cost(50);
  7113   format %{ "XOR    $dst,$dst" %}
  7114   opcode(0x33);  /* + rd */
  7115   ins_encode( OpcP, RegReg( dst, dst ) );
  7116   ins_pipe( ialu_reg );
  7117 %}
  7119 instruct loadConP(eRegP dst, immP src) %{
  7120   match(Set dst src);
  7122   format %{ "MOV    $dst,$src" %}
  7123   opcode(0xB8);  /* + rd */
  7124   ins_encode( LdImmP(dst, src) );
  7125   ins_pipe( ialu_reg_fat );
  7126 %}
  7128 instruct loadConL(eRegL dst, immL src, eFlagsReg cr) %{
  7129   match(Set dst src);
  7130   effect(KILL cr);
  7131   ins_cost(200);
  7132   format %{ "MOV    $dst.lo,$src.lo\n\t"
  7133             "MOV    $dst.hi,$src.hi" %}
  7134   opcode(0xB8);
  7135   ins_encode( LdImmL_Lo(dst, src), LdImmL_Hi(dst, src) );
  7136   ins_pipe( ialu_reg_long_fat );
  7137 %}
  7139 instruct loadConL0(eRegL dst, immL0 src, eFlagsReg cr) %{
  7140   match(Set dst src);
  7141   effect(KILL cr);
  7142   ins_cost(150);
  7143   format %{ "XOR    $dst.lo,$dst.lo\n\t"
  7144             "XOR    $dst.hi,$dst.hi" %}
  7145   opcode(0x33,0x33);
  7146   ins_encode( RegReg_Lo(dst,dst), RegReg_Hi(dst, dst) );
  7147   ins_pipe( ialu_reg_long );
  7148 %}
  7150 // The instruction usage is guarded by predicate in operand immF().
  7151 instruct loadConF(regF dst, immF con) %{
  7152   match(Set dst con);
  7153   ins_cost(125);
  7154   format %{ "FLD_S  ST,[$constantaddress]\t# load from constant table: float=$con\n\t"
  7155             "FSTP   $dst" %}
  7156   ins_encode %{
  7157     __ fld_s($constantaddress($con));
  7158     __ fstp_d($dst$$reg);
  7159   %}
  7160   ins_pipe(fpu_reg_con);
  7161 %}
  7163 // The instruction usage is guarded by predicate in operand immF0().
  7164 instruct loadConF0(regF dst, immF0 con) %{
  7165   match(Set dst con);
  7166   ins_cost(125);
  7167   format %{ "FLDZ   ST\n\t"
  7168             "FSTP   $dst" %}
  7169   ins_encode %{
  7170     __ fldz();
  7171     __ fstp_d($dst$$reg);
  7172   %}
  7173   ins_pipe(fpu_reg_con);
  7174 %}
  7176 // The instruction usage is guarded by predicate in operand immF1().
  7177 instruct loadConF1(regF dst, immF1 con) %{
  7178   match(Set dst con);
  7179   ins_cost(125);
  7180   format %{ "FLD1   ST\n\t"
  7181             "FSTP   $dst" %}
  7182   ins_encode %{
  7183     __ fld1();
  7184     __ fstp_d($dst$$reg);
  7185   %}
  7186   ins_pipe(fpu_reg_con);
  7187 %}
  7189 // The instruction usage is guarded by predicate in operand immXF().
  7190 instruct loadConX(regX dst, immXF con) %{
  7191   match(Set dst con);
  7192   ins_cost(125);
  7193   format %{ "MOVSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
  7194   ins_encode %{
  7195     __ movflt($dst$$XMMRegister, $constantaddress($con));
  7196   %}
  7197   ins_pipe(pipe_slow);
  7198 %}
  7200 // The instruction usage is guarded by predicate in operand immXF0().
  7201 instruct loadConX0(regX dst, immXF0 src) %{
  7202   match(Set dst src);
  7203   ins_cost(100);
  7204   format %{ "XORPS  $dst,$dst\t# float 0.0" %}
  7205   ins_encode %{
  7206     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
  7207   %}
  7208   ins_pipe(pipe_slow);
  7209 %}
  7211 // The instruction usage is guarded by predicate in operand immD().
  7212 instruct loadConD(regD dst, immD con) %{
  7213   match(Set dst con);
  7214   ins_cost(125);
  7216   format %{ "FLD_D  ST,[$constantaddress]\t# load from constant table: double=$con\n\t"
  7217             "FSTP   $dst" %}
  7218   ins_encode %{
  7219     __ fld_d($constantaddress($con));
  7220     __ fstp_d($dst$$reg);
  7221   %}
  7222   ins_pipe(fpu_reg_con);
  7223 %}
  7225 // The instruction usage is guarded by predicate in operand immD0().
  7226 instruct loadConD0(regD dst, immD0 con) %{
  7227   match(Set dst con);
  7228   ins_cost(125);
  7230   format %{ "FLDZ   ST\n\t"
  7231             "FSTP   $dst" %}
  7232   ins_encode %{
  7233     __ fldz();
  7234     __ fstp_d($dst$$reg);
  7235   %}
  7236   ins_pipe(fpu_reg_con);
  7237 %}
  7239 // The instruction usage is guarded by predicate in operand immD1().
  7240 instruct loadConD1(regD dst, immD1 con) %{
  7241   match(Set dst con);
  7242   ins_cost(125);
  7244   format %{ "FLD1   ST\n\t"
  7245             "FSTP   $dst" %}
  7246   ins_encode %{
  7247     __ fld1();
  7248     __ fstp_d($dst$$reg);
  7249   %}
  7250   ins_pipe(fpu_reg_con);
  7251 %}
  7253 // The instruction usage is guarded by predicate in operand immXD().
  7254 instruct loadConXD(regXD dst, immXD con) %{
  7255   match(Set dst con);
  7256   ins_cost(125);
  7257   format %{ "MOVSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
  7258   ins_encode %{
  7259     __ movdbl($dst$$XMMRegister, $constantaddress($con));
  7260   %}
  7261   ins_pipe(pipe_slow);
  7262 %}
  7264 // The instruction usage is guarded by predicate in operand immXD0().
  7265 instruct loadConXD0(regXD dst, immXD0 src) %{
  7266   match(Set dst src);
  7267   ins_cost(100);
  7268   format %{ "XORPD  $dst,$dst\t# double 0.0" %}
  7269   ins_encode( Opcode(0x66), Opcode(0x0F), Opcode(0x57), RegReg(dst,dst));
  7270   ins_pipe( pipe_slow );
  7271 %}
  7273 // Load Stack Slot
  7274 instruct loadSSI(eRegI dst, stackSlotI src) %{
  7275   match(Set dst src);
  7276   ins_cost(125);
  7278   format %{ "MOV    $dst,$src" %}
  7279   opcode(0x8B);
  7280   ins_encode( OpcP, RegMem(dst,src));
  7281   ins_pipe( ialu_reg_mem );
  7282 %}
  7284 instruct loadSSL(eRegL dst, stackSlotL src) %{
  7285   match(Set dst src);
  7287   ins_cost(200);
  7288   format %{ "MOV    $dst,$src.lo\n\t"
  7289             "MOV    $dst+4,$src.hi" %}
  7290   opcode(0x8B, 0x8B);
  7291   ins_encode( OpcP, RegMem( dst, src ), OpcS, RegMem_Hi( dst, src ) );
  7292   ins_pipe( ialu_mem_long_reg );
  7293 %}
  7295 // Load Stack Slot
  7296 instruct loadSSP(eRegP dst, stackSlotP src) %{
  7297   match(Set dst src);
  7298   ins_cost(125);
  7300   format %{ "MOV    $dst,$src" %}
  7301   opcode(0x8B);
  7302   ins_encode( OpcP, RegMem(dst,src));
  7303   ins_pipe( ialu_reg_mem );
  7304 %}
  7306 // Load Stack Slot
  7307 instruct loadSSF(regF dst, stackSlotF src) %{
  7308   match(Set dst src);
  7309   ins_cost(125);
  7311   format %{ "FLD_S  $src\n\t"
  7312             "FSTP   $dst" %}
  7313   opcode(0xD9);               /* D9 /0, FLD m32real */
  7314   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
  7315               Pop_Reg_F(dst) );
  7316   ins_pipe( fpu_reg_mem );
  7317 %}
  7319 // Load Stack Slot
  7320 instruct loadSSD(regD dst, stackSlotD src) %{
  7321   match(Set dst src);
  7322   ins_cost(125);
  7324   format %{ "FLD_D  $src\n\t"
  7325             "FSTP   $dst" %}
  7326   opcode(0xDD);               /* DD /0, FLD m64real */
  7327   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
  7328               Pop_Reg_D(dst) );
  7329   ins_pipe( fpu_reg_mem );
  7330 %}
  7332 // Prefetch instructions.
  7333 // Must be safe to execute with invalid address (cannot fault).
  7335 instruct prefetchr0( memory mem ) %{
  7336   predicate(UseSSE==0 && !VM_Version::supports_3dnow_prefetch());
  7337   match(PrefetchRead mem);
  7338   ins_cost(0);
  7339   size(0);
  7340   format %{ "PREFETCHR (non-SSE is empty encoding)" %}
  7341   ins_encode();
  7342   ins_pipe(empty);
  7343 %}
  7345 instruct prefetchr( memory mem ) %{
  7346   predicate(UseSSE==0 && VM_Version::supports_3dnow_prefetch() || ReadPrefetchInstr==3);
  7347   match(PrefetchRead mem);
  7348   ins_cost(100);
  7350   format %{ "PREFETCHR $mem\t! Prefetch into level 1 cache for read" %}
  7351   opcode(0x0F, 0x0d);     /* Opcode 0F 0d /0 */
  7352   ins_encode(OpcP, OpcS, RMopc_Mem(0x00,mem));
  7353   ins_pipe(ialu_mem);
  7354 %}
  7356 instruct prefetchrNTA( memory mem ) %{
  7357   predicate(UseSSE>=1 && ReadPrefetchInstr==0);
  7358   match(PrefetchRead mem);
  7359   ins_cost(100);
  7361   format %{ "PREFETCHNTA $mem\t! Prefetch into non-temporal cache for read" %}
  7362   opcode(0x0F, 0x18);     /* Opcode 0F 18 /0 */
  7363   ins_encode(OpcP, OpcS, RMopc_Mem(0x00,mem));
  7364   ins_pipe(ialu_mem);
  7365 %}
  7367 instruct prefetchrT0( memory mem ) %{
  7368   predicate(UseSSE>=1 && ReadPrefetchInstr==1);
  7369   match(PrefetchRead mem);
  7370   ins_cost(100);
  7372   format %{ "PREFETCHT0 $mem\t! Prefetch into L1 and L2 caches for read" %}
  7373   opcode(0x0F, 0x18);     /* Opcode 0F 18 /1 */
  7374   ins_encode(OpcP, OpcS, RMopc_Mem(0x01,mem));
  7375   ins_pipe(ialu_mem);
  7376 %}
  7378 instruct prefetchrT2( memory mem ) %{
  7379   predicate(UseSSE>=1 && ReadPrefetchInstr==2);
  7380   match(PrefetchRead mem);
  7381   ins_cost(100);
  7383   format %{ "PREFETCHT2 $mem\t! Prefetch into L2 cache for read" %}
  7384   opcode(0x0F, 0x18);     /* Opcode 0F 18 /3 */
  7385   ins_encode(OpcP, OpcS, RMopc_Mem(0x03,mem));
  7386   ins_pipe(ialu_mem);
  7387 %}
  7389 instruct prefetchw0( memory mem ) %{
  7390   predicate(UseSSE==0 && !VM_Version::supports_3dnow_prefetch());
  7391   match(PrefetchWrite mem);
  7392   ins_cost(0);
  7393   size(0);
  7394   format %{ "Prefetch (non-SSE is empty encoding)" %}
  7395   ins_encode();
  7396   ins_pipe(empty);
  7397 %}
  7399 instruct prefetchw( memory mem ) %{
  7400   predicate(UseSSE==0 && VM_Version::supports_3dnow_prefetch() || AllocatePrefetchInstr==3);
  7401   match( PrefetchWrite mem );
  7402   ins_cost(100);
  7404   format %{ "PREFETCHW $mem\t! Prefetch into L1 cache and mark modified" %}
  7405   opcode(0x0F, 0x0D);     /* Opcode 0F 0D /1 */
  7406   ins_encode(OpcP, OpcS, RMopc_Mem(0x01,mem));
  7407   ins_pipe(ialu_mem);
  7408 %}
  7410 instruct prefetchwNTA( memory mem ) %{
  7411   predicate(UseSSE>=1 && AllocatePrefetchInstr==0);
  7412   match(PrefetchWrite mem);
  7413   ins_cost(100);
  7415   format %{ "PREFETCHNTA $mem\t! Prefetch into non-temporal cache for write" %}
  7416   opcode(0x0F, 0x18);     /* Opcode 0F 18 /0 */
  7417   ins_encode(OpcP, OpcS, RMopc_Mem(0x00,mem));
  7418   ins_pipe(ialu_mem);
  7419 %}
  7421 instruct prefetchwT0( memory mem ) %{
  7422   predicate(UseSSE>=1 && AllocatePrefetchInstr==1);
  7423   match(PrefetchWrite mem);
  7424   ins_cost(100);
  7426   format %{ "PREFETCHT0 $mem\t! Prefetch into L1 and L2 caches for write" %}
  7427   opcode(0x0F, 0x18);     /* Opcode 0F 18 /1 */
  7428   ins_encode(OpcP, OpcS, RMopc_Mem(0x01,mem));
  7429   ins_pipe(ialu_mem);
  7430 %}
  7432 instruct prefetchwT2( memory mem ) %{
  7433   predicate(UseSSE>=1 && AllocatePrefetchInstr==2);
  7434   match(PrefetchWrite mem);
  7435   ins_cost(100);
  7437   format %{ "PREFETCHT2 $mem\t! Prefetch into L2 cache for write" %}
  7438   opcode(0x0F, 0x18);     /* Opcode 0F 18 /3 */
  7439   ins_encode(OpcP, OpcS, RMopc_Mem(0x03,mem));
  7440   ins_pipe(ialu_mem);
  7441 %}
  7443 //----------Store Instructions-------------------------------------------------
  7445 // Store Byte
  7446 instruct storeB(memory mem, xRegI src) %{
  7447   match(Set mem (StoreB mem src));
  7449   ins_cost(125);
  7450   format %{ "MOV8   $mem,$src" %}
  7451   opcode(0x88);
  7452   ins_encode( OpcP, RegMem( src, mem ) );
  7453   ins_pipe( ialu_mem_reg );
  7454 %}
  7456 // Store Char/Short
  7457 instruct storeC(memory mem, eRegI src) %{
  7458   match(Set mem (StoreC mem src));
  7460   ins_cost(125);
  7461   format %{ "MOV16  $mem,$src" %}
  7462   opcode(0x89, 0x66);
  7463   ins_encode( OpcS, OpcP, RegMem( src, mem ) );
  7464   ins_pipe( ialu_mem_reg );
  7465 %}
  7467 // Store Integer
  7468 instruct storeI(memory mem, eRegI src) %{
  7469   match(Set mem (StoreI mem src));
  7471   ins_cost(125);
  7472   format %{ "MOV    $mem,$src" %}
  7473   opcode(0x89);
  7474   ins_encode( OpcP, RegMem( src, mem ) );
  7475   ins_pipe( ialu_mem_reg );
  7476 %}
  7478 // Store Long
  7479 instruct storeL(long_memory mem, eRegL src) %{
  7480   predicate(!((StoreLNode*)n)->require_atomic_access());
  7481   match(Set mem (StoreL mem src));
  7483   ins_cost(200);
  7484   format %{ "MOV    $mem,$src.lo\n\t"
  7485             "MOV    $mem+4,$src.hi" %}
  7486   opcode(0x89, 0x89);
  7487   ins_encode( OpcP, RegMem( src, mem ), OpcS, RegMem_Hi( src, mem ) );
  7488   ins_pipe( ialu_mem_long_reg );
  7489 %}
  7491 // Store Long to Integer
  7492 instruct storeL2I(memory mem, eRegL src) %{
  7493   match(Set mem (StoreI mem (ConvL2I src)));
  7495   format %{ "MOV    $mem,$src.lo\t# long -> int" %}
  7496   ins_encode %{
  7497     __ movl($mem$$Address, $src$$Register);
  7498   %}
  7499   ins_pipe(ialu_mem_reg);
  7500 %}
  7502 // Volatile Store Long.  Must be atomic, so move it into
  7503 // the FP TOS and then do a 64-bit FIST.  Has to probe the
  7504 // target address before the store (for null-ptr checks)
  7505 // so the memory operand is used twice in the encoding.
  7506 instruct storeL_volatile(memory mem, stackSlotL src, eFlagsReg cr ) %{
  7507   predicate(UseSSE<=1 && ((StoreLNode*)n)->require_atomic_access());
  7508   match(Set mem (StoreL mem src));
  7509   effect( KILL cr );
  7510   ins_cost(400);
  7511   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
  7512             "FILD   $src\n\t"
  7513             "FISTp  $mem\t # 64-bit atomic volatile long store" %}
  7514   opcode(0x3B);
  7515   ins_encode( OpcP, RegMem( EAX, mem ), enc_storeL_volatile(mem,src));
  7516   ins_pipe( fpu_reg_mem );
  7517 %}
  7519 instruct storeLX_volatile(memory mem, stackSlotL src, regXD tmp, eFlagsReg cr) %{
  7520   predicate(UseSSE>=2 && ((StoreLNode*)n)->require_atomic_access());
  7521   match(Set mem (StoreL mem src));
  7522   effect( TEMP tmp, KILL cr );
  7523   ins_cost(380);
  7524   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
  7525             "MOVSD  $tmp,$src\n\t"
  7526             "MOVSD  $mem,$tmp\t # 64-bit atomic volatile long store" %}
  7527   opcode(0x3B);
  7528   ins_encode( OpcP, RegMem( EAX, mem ), enc_storeLX_volatile(mem, src, tmp));
  7529   ins_pipe( pipe_slow );
  7530 %}
  7532 instruct storeLX_reg_volatile(memory mem, eRegL src, regXD tmp2, regXD tmp, eFlagsReg cr) %{
  7533   predicate(UseSSE>=2 && ((StoreLNode*)n)->require_atomic_access());
  7534   match(Set mem (StoreL mem src));
  7535   effect( TEMP tmp2 , TEMP tmp, KILL cr );
  7536   ins_cost(360);
  7537   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
  7538             "MOVD   $tmp,$src.lo\n\t"
  7539             "MOVD   $tmp2,$src.hi\n\t"
  7540             "PUNPCKLDQ $tmp,$tmp2\n\t"
  7541             "MOVSD  $mem,$tmp\t # 64-bit atomic volatile long store" %}
  7542   opcode(0x3B);
  7543   ins_encode( OpcP, RegMem( EAX, mem ), enc_storeLX_reg_volatile(mem, src, tmp, tmp2));
  7544   ins_pipe( pipe_slow );
  7545 %}
  7547 // Store Pointer; for storing unknown oops and raw pointers
  7548 instruct storeP(memory mem, anyRegP src) %{
  7549   match(Set mem (StoreP mem src));
  7551   ins_cost(125);
  7552   format %{ "MOV    $mem,$src" %}
  7553   opcode(0x89);
  7554   ins_encode( OpcP, RegMem( src, mem ) );
  7555   ins_pipe( ialu_mem_reg );
  7556 %}
  7558 // Store Integer Immediate
  7559 instruct storeImmI(memory mem, immI src) %{
  7560   match(Set mem (StoreI mem src));
  7562   ins_cost(150);
  7563   format %{ "MOV    $mem,$src" %}
  7564   opcode(0xC7);               /* C7 /0 */
  7565   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32( src ));
  7566   ins_pipe( ialu_mem_imm );
  7567 %}
  7569 // Store Short/Char Immediate
  7570 instruct storeImmI16(memory mem, immI16 src) %{
  7571   predicate(UseStoreImmI16);
  7572   match(Set mem (StoreC mem src));
  7574   ins_cost(150);
  7575   format %{ "MOV16  $mem,$src" %}
  7576   opcode(0xC7);     /* C7 /0 Same as 32 store immediate with prefix */
  7577   ins_encode( SizePrefix, OpcP, RMopc_Mem(0x00,mem),  Con16( src ));
  7578   ins_pipe( ialu_mem_imm );
  7579 %}
  7581 // Store Pointer Immediate; null pointers or constant oops that do not
  7582 // need card-mark barriers.
  7583 instruct storeImmP(memory mem, immP src) %{
  7584   match(Set mem (StoreP mem src));
  7586   ins_cost(150);
  7587   format %{ "MOV    $mem,$src" %}
  7588   opcode(0xC7);               /* C7 /0 */
  7589   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32( src ));
  7590   ins_pipe( ialu_mem_imm );
  7591 %}
  7593 // Store Byte Immediate
  7594 instruct storeImmB(memory mem, immI8 src) %{
  7595   match(Set mem (StoreB mem src));
  7597   ins_cost(150);
  7598   format %{ "MOV8   $mem,$src" %}
  7599   opcode(0xC6);               /* C6 /0 */
  7600   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con8or32( src ));
  7601   ins_pipe( ialu_mem_imm );
  7602 %}
  7604 // Store Aligned Packed Byte XMM register to memory
  7605 instruct storeA8B(memory mem, regXD src) %{
  7606   predicate(UseSSE>=1);
  7607   match(Set mem (Store8B mem src));
  7608   ins_cost(145);
  7609   format %{ "MOVQ  $mem,$src\t! packed8B" %}
  7610   ins_encode( movq_st(mem, src));
  7611   ins_pipe( pipe_slow );
  7612 %}
  7614 // Store Aligned Packed Char/Short XMM register to memory
  7615 instruct storeA4C(memory mem, regXD src) %{
  7616   predicate(UseSSE>=1);
  7617   match(Set mem (Store4C mem src));
  7618   ins_cost(145);
  7619   format %{ "MOVQ  $mem,$src\t! packed4C" %}
  7620   ins_encode( movq_st(mem, src));
  7621   ins_pipe( pipe_slow );
  7622 %}
  7624 // Store Aligned Packed Integer XMM register to memory
  7625 instruct storeA2I(memory mem, regXD src) %{
  7626   predicate(UseSSE>=1);
  7627   match(Set mem (Store2I mem src));
  7628   ins_cost(145);
  7629   format %{ "MOVQ  $mem,$src\t! packed2I" %}
  7630   ins_encode( movq_st(mem, src));
  7631   ins_pipe( pipe_slow );
  7632 %}
  7634 // Store CMS card-mark Immediate
  7635 instruct storeImmCM(memory mem, immI8 src) %{
  7636   match(Set mem (StoreCM mem src));
  7638   ins_cost(150);
  7639   format %{ "MOV8   $mem,$src\t! CMS card-mark imm0" %}
  7640   opcode(0xC6);               /* C6 /0 */
  7641   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con8or32( src ));
  7642   ins_pipe( ialu_mem_imm );
  7643 %}
  7645 // Store Double
  7646 instruct storeD( memory mem, regDPR1 src) %{
  7647   predicate(UseSSE<=1);
  7648   match(Set mem (StoreD mem src));
  7650   ins_cost(100);
  7651   format %{ "FST_D  $mem,$src" %}
  7652   opcode(0xDD);       /* DD /2 */
  7653   ins_encode( enc_FP_store(mem,src) );
  7654   ins_pipe( fpu_mem_reg );
  7655 %}
  7657 // Store double does rounding on x86
  7658 instruct storeD_rounded( memory mem, regDPR1 src) %{
  7659   predicate(UseSSE<=1);
  7660   match(Set mem (StoreD mem (RoundDouble src)));
  7662   ins_cost(100);
  7663   format %{ "FST_D  $mem,$src\t# round" %}
  7664   opcode(0xDD);       /* DD /2 */
  7665   ins_encode( enc_FP_store(mem,src) );
  7666   ins_pipe( fpu_mem_reg );
  7667 %}
  7669 // Store XMM register to memory (double-precision floating points)
  7670 // MOVSD instruction
  7671 instruct storeXD(memory mem, regXD src) %{
  7672   predicate(UseSSE>=2);
  7673   match(Set mem (StoreD mem src));
  7674   ins_cost(95);
  7675   format %{ "MOVSD  $mem,$src" %}
  7676   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x11), RegMem(src, mem));
  7677   ins_pipe( pipe_slow );
  7678 %}
  7680 // Store XMM register to memory (single-precision floating point)
  7681 // MOVSS instruction
  7682 instruct storeX(memory mem, regX src) %{
  7683   predicate(UseSSE>=1);
  7684   match(Set mem (StoreF mem src));
  7685   ins_cost(95);
  7686   format %{ "MOVSS  $mem,$src" %}
  7687   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x11), RegMem(src, mem));
  7688   ins_pipe( pipe_slow );
  7689 %}
  7691 // Store Aligned Packed Single Float XMM register to memory
  7692 instruct storeA2F(memory mem, regXD src) %{
  7693   predicate(UseSSE>=1);
  7694   match(Set mem (Store2F mem src));
  7695   ins_cost(145);
  7696   format %{ "MOVQ  $mem,$src\t! packed2F" %}
  7697   ins_encode( movq_st(mem, src));
  7698   ins_pipe( pipe_slow );
  7699 %}
  7701 // Store Float
  7702 instruct storeF( memory mem, regFPR1 src) %{
  7703   predicate(UseSSE==0);
  7704   match(Set mem (StoreF mem src));
  7706   ins_cost(100);
  7707   format %{ "FST_S  $mem,$src" %}
  7708   opcode(0xD9);       /* D9 /2 */
  7709   ins_encode( enc_FP_store(mem,src) );
  7710   ins_pipe( fpu_mem_reg );
  7711 %}
  7713 // Store Float does rounding on x86
  7714 instruct storeF_rounded( memory mem, regFPR1 src) %{
  7715   predicate(UseSSE==0);
  7716   match(Set mem (StoreF mem (RoundFloat src)));
  7718   ins_cost(100);
  7719   format %{ "FST_S  $mem,$src\t# round" %}
  7720   opcode(0xD9);       /* D9 /2 */
  7721   ins_encode( enc_FP_store(mem,src) );
  7722   ins_pipe( fpu_mem_reg );
  7723 %}
  7725 // Store Float does rounding on x86
  7726 instruct storeF_Drounded( memory mem, regDPR1 src) %{
  7727   predicate(UseSSE<=1);
  7728   match(Set mem (StoreF mem (ConvD2F src)));
  7730   ins_cost(100);
  7731   format %{ "FST_S  $mem,$src\t# D-round" %}
  7732   opcode(0xD9);       /* D9 /2 */
  7733   ins_encode( enc_FP_store(mem,src) );
  7734   ins_pipe( fpu_mem_reg );
  7735 %}
  7737 // Store immediate Float value (it is faster than store from FPU register)
  7738 // The instruction usage is guarded by predicate in operand immF().
  7739 instruct storeF_imm( memory mem, immF src) %{
  7740   match(Set mem (StoreF mem src));
  7742   ins_cost(50);
  7743   format %{ "MOV    $mem,$src\t# store float" %}
  7744   opcode(0xC7);               /* C7 /0 */
  7745   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32F_as_bits( src ));
  7746   ins_pipe( ialu_mem_imm );
  7747 %}
  7749 // Store immediate Float value (it is faster than store from XMM register)
  7750 // The instruction usage is guarded by predicate in operand immXF().
  7751 instruct storeX_imm( memory mem, immXF src) %{
  7752   match(Set mem (StoreF mem src));
  7754   ins_cost(50);
  7755   format %{ "MOV    $mem,$src\t# store float" %}
  7756   opcode(0xC7);               /* C7 /0 */
  7757   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32XF_as_bits( src ));
  7758   ins_pipe( ialu_mem_imm );
  7759 %}
  7761 // Store Integer to stack slot
  7762 instruct storeSSI(stackSlotI dst, eRegI src) %{
  7763   match(Set dst src);
  7765   ins_cost(100);
  7766   format %{ "MOV    $dst,$src" %}
  7767   opcode(0x89);
  7768   ins_encode( OpcPRegSS( dst, src ) );
  7769   ins_pipe( ialu_mem_reg );
  7770 %}
  7772 // Store Integer to stack slot
  7773 instruct storeSSP(stackSlotP dst, eRegP src) %{
  7774   match(Set dst src);
  7776   ins_cost(100);
  7777   format %{ "MOV    $dst,$src" %}
  7778   opcode(0x89);
  7779   ins_encode( OpcPRegSS( dst, src ) );
  7780   ins_pipe( ialu_mem_reg );
  7781 %}
  7783 // Store Long to stack slot
  7784 instruct storeSSL(stackSlotL dst, eRegL src) %{
  7785   match(Set dst src);
  7787   ins_cost(200);
  7788   format %{ "MOV    $dst,$src.lo\n\t"
  7789             "MOV    $dst+4,$src.hi" %}
  7790   opcode(0x89, 0x89);
  7791   ins_encode( OpcP, RegMem( src, dst ), OpcS, RegMem_Hi( src, dst ) );
  7792   ins_pipe( ialu_mem_long_reg );
  7793 %}
  7795 //----------MemBar Instructions-----------------------------------------------
  7796 // Memory barrier flavors
  7798 instruct membar_acquire() %{
  7799   match(MemBarAcquire);
  7800   ins_cost(400);
  7802   size(0);
  7803   format %{ "MEMBAR-acquire ! (empty encoding)" %}
  7804   ins_encode();
  7805   ins_pipe(empty);
  7806 %}
  7808 instruct membar_acquire_lock() %{
  7809   match(MemBarAcquire);
  7810   predicate(Matcher::prior_fast_lock(n));
  7811   ins_cost(0);
  7813   size(0);
  7814   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
  7815   ins_encode( );
  7816   ins_pipe(empty);
  7817 %}
  7819 instruct membar_release() %{
  7820   match(MemBarRelease);
  7821   ins_cost(400);
  7823   size(0);
  7824   format %{ "MEMBAR-release ! (empty encoding)" %}
  7825   ins_encode( );
  7826   ins_pipe(empty);
  7827 %}
  7829 instruct membar_release_lock() %{
  7830   match(MemBarRelease);
  7831   predicate(Matcher::post_fast_unlock(n));
  7832   ins_cost(0);
  7834   size(0);
  7835   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
  7836   ins_encode( );
  7837   ins_pipe(empty);
  7838 %}
  7840 instruct membar_volatile(eFlagsReg cr) %{
  7841   match(MemBarVolatile);
  7842   effect(KILL cr);
  7843   ins_cost(400);
  7845   format %{ 
  7846     $$template
  7847     if (os::is_MP()) {
  7848       $$emit$$"LOCK ADDL [ESP + #0], 0\t! membar_volatile"
  7849     } else {
  7850       $$emit$$"MEMBAR-volatile ! (empty encoding)"
  7852   %}
  7853   ins_encode %{
  7854     __ membar(Assembler::StoreLoad);
  7855   %}
  7856   ins_pipe(pipe_slow);
  7857 %}
  7859 instruct unnecessary_membar_volatile() %{
  7860   match(MemBarVolatile);
  7861   predicate(Matcher::post_store_load_barrier(n));
  7862   ins_cost(0);
  7864   size(0);
  7865   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
  7866   ins_encode( );
  7867   ins_pipe(empty);
  7868 %}
  7870 //----------Move Instructions--------------------------------------------------
  7871 instruct castX2P(eAXRegP dst, eAXRegI src) %{
  7872   match(Set dst (CastX2P src));
  7873   format %{ "# X2P  $dst, $src" %}
  7874   ins_encode( /*empty encoding*/ );
  7875   ins_cost(0);
  7876   ins_pipe(empty);
  7877 %}
  7879 instruct castP2X(eRegI dst, eRegP src ) %{
  7880   match(Set dst (CastP2X src));
  7881   ins_cost(50);
  7882   format %{ "MOV    $dst, $src\t# CastP2X" %}
  7883   ins_encode( enc_Copy( dst, src) );
  7884   ins_pipe( ialu_reg_reg );
  7885 %}
  7887 //----------Conditional Move---------------------------------------------------
  7888 // Conditional move
  7889 instruct cmovI_reg(eRegI dst, eRegI src, eFlagsReg cr, cmpOp cop ) %{
  7890   predicate(VM_Version::supports_cmov() );
  7891   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
  7892   ins_cost(200);
  7893   format %{ "CMOV$cop $dst,$src" %}
  7894   opcode(0x0F,0x40);
  7895   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
  7896   ins_pipe( pipe_cmov_reg );
  7897 %}
  7899 instruct cmovI_regU( cmpOpU cop, eFlagsRegU cr, eRegI dst, eRegI src ) %{
  7900   predicate(VM_Version::supports_cmov() );
  7901   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
  7902   ins_cost(200);
  7903   format %{ "CMOV$cop $dst,$src" %}
  7904   opcode(0x0F,0x40);
  7905   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
  7906   ins_pipe( pipe_cmov_reg );
  7907 %}
  7909 instruct cmovI_regUCF( cmpOpUCF cop, eFlagsRegUCF cr, eRegI dst, eRegI src ) %{
  7910   predicate(VM_Version::supports_cmov() );
  7911   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
  7912   ins_cost(200);
  7913   expand %{
  7914     cmovI_regU(cop, cr, dst, src);
  7915   %}
  7916 %}
  7918 // Conditional move
  7919 instruct cmovI_mem(cmpOp cop, eFlagsReg cr, eRegI dst, memory src) %{
  7920   predicate(VM_Version::supports_cmov() );
  7921   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
  7922   ins_cost(250);
  7923   format %{ "CMOV$cop $dst,$src" %}
  7924   opcode(0x0F,0x40);
  7925   ins_encode( enc_cmov(cop), RegMem( dst, src ) );
  7926   ins_pipe( pipe_cmov_mem );
  7927 %}
  7929 // Conditional move
  7930 instruct cmovI_memU(cmpOpU cop, eFlagsRegU cr, eRegI dst, memory src) %{
  7931   predicate(VM_Version::supports_cmov() );
  7932   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
  7933   ins_cost(250);
  7934   format %{ "CMOV$cop $dst,$src" %}
  7935   opcode(0x0F,0x40);
  7936   ins_encode( enc_cmov(cop), RegMem( dst, src ) );
  7937   ins_pipe( pipe_cmov_mem );
  7938 %}
  7940 instruct cmovI_memUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegI dst, memory src) %{
  7941   predicate(VM_Version::supports_cmov() );
  7942   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
  7943   ins_cost(250);
  7944   expand %{
  7945     cmovI_memU(cop, cr, dst, src);
  7946   %}
  7947 %}
  7949 // Conditional move
  7950 instruct cmovP_reg(eRegP dst, eRegP src, eFlagsReg cr, cmpOp cop ) %{
  7951   predicate(VM_Version::supports_cmov() );
  7952   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
  7953   ins_cost(200);
  7954   format %{ "CMOV$cop $dst,$src\t# ptr" %}
  7955   opcode(0x0F,0x40);
  7956   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
  7957   ins_pipe( pipe_cmov_reg );
  7958 %}
  7960 // Conditional move (non-P6 version)
  7961 // Note:  a CMoveP is generated for  stubs and native wrappers
  7962 //        regardless of whether we are on a P6, so we
  7963 //        emulate a cmov here
  7964 instruct cmovP_reg_nonP6(eRegP dst, eRegP src, eFlagsReg cr, cmpOp cop ) %{
  7965   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
  7966   ins_cost(300);
  7967   format %{ "Jn$cop   skip\n\t"
  7968           "MOV    $dst,$src\t# pointer\n"
  7969       "skip:" %}
  7970   opcode(0x8b);
  7971   ins_encode( enc_cmov_branch(cop, 0x2), OpcP, RegReg(dst, src));
  7972   ins_pipe( pipe_cmov_reg );
  7973 %}
  7975 // Conditional move
  7976 instruct cmovP_regU(cmpOpU cop, eFlagsRegU cr, eRegP dst, eRegP src ) %{
  7977   predicate(VM_Version::supports_cmov() );
  7978   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
  7979   ins_cost(200);
  7980   format %{ "CMOV$cop $dst,$src\t# ptr" %}
  7981   opcode(0x0F,0x40);
  7982   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
  7983   ins_pipe( pipe_cmov_reg );
  7984 %}
  7986 instruct cmovP_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegP dst, eRegP src ) %{
  7987   predicate(VM_Version::supports_cmov() );
  7988   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
  7989   ins_cost(200);
  7990   expand %{
  7991     cmovP_regU(cop, cr, dst, src);
  7992   %}
  7993 %}
  7995 // DISABLED: Requires the ADLC to emit a bottom_type call that
  7996 // correctly meets the two pointer arguments; one is an incoming
  7997 // register but the other is a memory operand.  ALSO appears to
  7998 // be buggy with implicit null checks.
  7999 //
  8000 //// Conditional move
  8001 //instruct cmovP_mem(cmpOp cop, eFlagsReg cr, eRegP dst, memory src) %{
  8002 //  predicate(VM_Version::supports_cmov() );
  8003 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
  8004 //  ins_cost(250);
  8005 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
  8006 //  opcode(0x0F,0x40);
  8007 //  ins_encode( enc_cmov(cop), RegMem( dst, src ) );
  8008 //  ins_pipe( pipe_cmov_mem );
  8009 //%}
  8010 //
  8011 //// Conditional move
  8012 //instruct cmovP_memU(cmpOpU cop, eFlagsRegU cr, eRegP dst, memory src) %{
  8013 //  predicate(VM_Version::supports_cmov() );
  8014 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
  8015 //  ins_cost(250);
  8016 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
  8017 //  opcode(0x0F,0x40);
  8018 //  ins_encode( enc_cmov(cop), RegMem( dst, src ) );
  8019 //  ins_pipe( pipe_cmov_mem );
  8020 //%}
  8022 // Conditional move
  8023 instruct fcmovD_regU(cmpOp_fcmov cop, eFlagsRegU cr, regDPR1 dst, regD src) %{
  8024   predicate(UseSSE<=1);
  8025   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
  8026   ins_cost(200);
  8027   format %{ "FCMOV$cop $dst,$src\t# double" %}
  8028   opcode(0xDA);
  8029   ins_encode( enc_cmov_d(cop,src) );
  8030   ins_pipe( pipe_cmovD_reg );
  8031 %}
  8033 // Conditional move
  8034 instruct fcmovF_regU(cmpOp_fcmov cop, eFlagsRegU cr, regFPR1 dst, regF src) %{
  8035   predicate(UseSSE==0);
  8036   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
  8037   ins_cost(200);
  8038   format %{ "FCMOV$cop $dst,$src\t# float" %}
  8039   opcode(0xDA);
  8040   ins_encode( enc_cmov_d(cop,src) );
  8041   ins_pipe( pipe_cmovD_reg );
  8042 %}
  8044 // Float CMOV on Intel doesn't handle *signed* compares, only unsigned.
  8045 instruct fcmovD_regS(cmpOp cop, eFlagsReg cr, regD dst, regD src) %{
  8046   predicate(UseSSE<=1);
  8047   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
  8048   ins_cost(200);
  8049   format %{ "Jn$cop   skip\n\t"
  8050             "MOV    $dst,$src\t# double\n"
  8051       "skip:" %}
  8052   opcode (0xdd, 0x3);     /* DD D8+i or DD /3 */
  8053   ins_encode( enc_cmov_branch( cop, 0x4 ), Push_Reg_D(src), OpcP, RegOpc(dst) );
  8054   ins_pipe( pipe_cmovD_reg );
  8055 %}
  8057 // Float CMOV on Intel doesn't handle *signed* compares, only unsigned.
  8058 instruct fcmovF_regS(cmpOp cop, eFlagsReg cr, regF dst, regF src) %{
  8059   predicate(UseSSE==0);
  8060   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
  8061   ins_cost(200);
  8062   format %{ "Jn$cop    skip\n\t"
  8063             "MOV    $dst,$src\t# float\n"
  8064       "skip:" %}
  8065   opcode (0xdd, 0x3);     /* DD D8+i or DD /3 */
  8066   ins_encode( enc_cmov_branch( cop, 0x4 ), Push_Reg_F(src), OpcP, RegOpc(dst) );
  8067   ins_pipe( pipe_cmovD_reg );
  8068 %}
  8070 // No CMOVE with SSE/SSE2
  8071 instruct fcmovX_regS(cmpOp cop, eFlagsReg cr, regX dst, regX src) %{
  8072   predicate (UseSSE>=1);
  8073   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
  8074   ins_cost(200);
  8075   format %{ "Jn$cop   skip\n\t"
  8076             "MOVSS  $dst,$src\t# float\n"
  8077       "skip:" %}
  8078   ins_encode %{
  8079     Label skip;
  8080     // Invert sense of branch from sense of CMOV
  8081     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
  8082     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
  8083     __ bind(skip);
  8084   %}
  8085   ins_pipe( pipe_slow );
  8086 %}
  8088 // No CMOVE with SSE/SSE2
  8089 instruct fcmovXD_regS(cmpOp cop, eFlagsReg cr, regXD dst, regXD src) %{
  8090   predicate (UseSSE>=2);
  8091   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
  8092   ins_cost(200);
  8093   format %{ "Jn$cop   skip\n\t"
  8094             "MOVSD  $dst,$src\t# float\n"
  8095       "skip:" %}
  8096   ins_encode %{
  8097     Label skip;
  8098     // Invert sense of branch from sense of CMOV
  8099     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
  8100     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
  8101     __ bind(skip);
  8102   %}
  8103   ins_pipe( pipe_slow );
  8104 %}
  8106 // unsigned version
  8107 instruct fcmovX_regU(cmpOpU cop, eFlagsRegU cr, regX dst, regX src) %{
  8108   predicate (UseSSE>=1);
  8109   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
  8110   ins_cost(200);
  8111   format %{ "Jn$cop   skip\n\t"
  8112             "MOVSS  $dst,$src\t# float\n"
  8113       "skip:" %}
  8114   ins_encode %{
  8115     Label skip;
  8116     // Invert sense of branch from sense of CMOV
  8117     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
  8118     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
  8119     __ bind(skip);
  8120   %}
  8121   ins_pipe( pipe_slow );
  8122 %}
  8124 instruct fcmovX_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, regX dst, regX src) %{
  8125   predicate (UseSSE>=1);
  8126   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
  8127   ins_cost(200);
  8128   expand %{
  8129     fcmovX_regU(cop, cr, dst, src);
  8130   %}
  8131 %}
  8133 // unsigned version
  8134 instruct fcmovXD_regU(cmpOpU cop, eFlagsRegU cr, regXD dst, regXD src) %{
  8135   predicate (UseSSE>=2);
  8136   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
  8137   ins_cost(200);
  8138   format %{ "Jn$cop   skip\n\t"
  8139             "MOVSD  $dst,$src\t# float\n"
  8140       "skip:" %}
  8141   ins_encode %{
  8142     Label skip;
  8143     // Invert sense of branch from sense of CMOV
  8144     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
  8145     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
  8146     __ bind(skip);
  8147   %}
  8148   ins_pipe( pipe_slow );
  8149 %}
  8151 instruct fcmovXD_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, regXD dst, regXD src) %{
  8152   predicate (UseSSE>=2);
  8153   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
  8154   ins_cost(200);
  8155   expand %{
  8156     fcmovXD_regU(cop, cr, dst, src);
  8157   %}
  8158 %}
  8160 instruct cmovL_reg(cmpOp cop, eFlagsReg cr, eRegL dst, eRegL src) %{
  8161   predicate(VM_Version::supports_cmov() );
  8162   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
  8163   ins_cost(200);
  8164   format %{ "CMOV$cop $dst.lo,$src.lo\n\t"
  8165             "CMOV$cop $dst.hi,$src.hi" %}
  8166   opcode(0x0F,0x40);
  8167   ins_encode( enc_cmov(cop), RegReg_Lo2( dst, src ), enc_cmov(cop), RegReg_Hi2( dst, src ) );
  8168   ins_pipe( pipe_cmov_reg_long );
  8169 %}
  8171 instruct cmovL_regU(cmpOpU cop, eFlagsRegU cr, eRegL dst, eRegL src) %{
  8172   predicate(VM_Version::supports_cmov() );
  8173   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
  8174   ins_cost(200);
  8175   format %{ "CMOV$cop $dst.lo,$src.lo\n\t"
  8176             "CMOV$cop $dst.hi,$src.hi" %}
  8177   opcode(0x0F,0x40);
  8178   ins_encode( enc_cmov(cop), RegReg_Lo2( dst, src ), enc_cmov(cop), RegReg_Hi2( dst, src ) );
  8179   ins_pipe( pipe_cmov_reg_long );
  8180 %}
  8182 instruct cmovL_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegL dst, eRegL src) %{
  8183   predicate(VM_Version::supports_cmov() );
  8184   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
  8185   ins_cost(200);
  8186   expand %{
  8187     cmovL_regU(cop, cr, dst, src);
  8188   %}
  8189 %}
  8191 //----------Arithmetic Instructions--------------------------------------------
  8192 //----------Addition Instructions----------------------------------------------
  8193 // Integer Addition Instructions
  8194 instruct addI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
  8195   match(Set dst (AddI dst src));
  8196   effect(KILL cr);
  8198   size(2);
  8199   format %{ "ADD    $dst,$src" %}
  8200   opcode(0x03);
  8201   ins_encode( OpcP, RegReg( dst, src) );
  8202   ins_pipe( ialu_reg_reg );
  8203 %}
  8205 instruct addI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
  8206   match(Set dst (AddI dst src));
  8207   effect(KILL cr);
  8209   format %{ "ADD    $dst,$src" %}
  8210   opcode(0x81, 0x00); /* /0 id */
  8211   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
  8212   ins_pipe( ialu_reg );
  8213 %}
  8215 instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
  8216   predicate(UseIncDec);
  8217   match(Set dst (AddI dst src));
  8218   effect(KILL cr);
  8220   size(1);
  8221   format %{ "INC    $dst" %}
  8222   opcode(0x40); /*  */
  8223   ins_encode( Opc_plus( primary, dst ) );
  8224   ins_pipe( ialu_reg );
  8225 %}
  8227 instruct leaI_eReg_immI(eRegI dst, eRegI src0, immI src1) %{
  8228   match(Set dst (AddI src0 src1));
  8229   ins_cost(110);
  8231   format %{ "LEA    $dst,[$src0 + $src1]" %}
  8232   opcode(0x8D); /* 0x8D /r */
  8233   ins_encode( OpcP, RegLea( dst, src0, src1 ) );
  8234   ins_pipe( ialu_reg_reg );
  8235 %}
  8237 instruct leaP_eReg_immI(eRegP dst, eRegP src0, immI src1) %{
  8238   match(Set dst (AddP src0 src1));
  8239   ins_cost(110);
  8241   format %{ "LEA    $dst,[$src0 + $src1]\t# ptr" %}
  8242   opcode(0x8D); /* 0x8D /r */
  8243   ins_encode( OpcP, RegLea( dst, src0, src1 ) );
  8244   ins_pipe( ialu_reg_reg );
  8245 %}
  8247 instruct decI_eReg(eRegI dst, immI_M1 src, eFlagsReg cr) %{
  8248   predicate(UseIncDec);
  8249   match(Set dst (AddI dst src));
  8250   effect(KILL cr);
  8252   size(1);
  8253   format %{ "DEC    $dst" %}
  8254   opcode(0x48); /*  */
  8255   ins_encode( Opc_plus( primary, dst ) );
  8256   ins_pipe( ialu_reg );
  8257 %}
  8259 instruct addP_eReg(eRegP dst, eRegI src, eFlagsReg cr) %{
  8260   match(Set dst (AddP dst src));
  8261   effect(KILL cr);
  8263   size(2);
  8264   format %{ "ADD    $dst,$src" %}
  8265   opcode(0x03);
  8266   ins_encode( OpcP, RegReg( dst, src) );
  8267   ins_pipe( ialu_reg_reg );
  8268 %}
  8270 instruct addP_eReg_imm(eRegP dst, immI src, eFlagsReg cr) %{
  8271   match(Set dst (AddP dst src));
  8272   effect(KILL cr);
  8274   format %{ "ADD    $dst,$src" %}
  8275   opcode(0x81,0x00); /* Opcode 81 /0 id */
  8276   // ins_encode( RegImm( dst, src) );
  8277   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
  8278   ins_pipe( ialu_reg );
  8279 %}
  8281 instruct addI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
  8282   match(Set dst (AddI dst (LoadI src)));
  8283   effect(KILL cr);
  8285   ins_cost(125);
  8286   format %{ "ADD    $dst,$src" %}
  8287   opcode(0x03);
  8288   ins_encode( OpcP, RegMem( dst, src) );
  8289   ins_pipe( ialu_reg_mem );
  8290 %}
  8292 instruct addI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
  8293   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
  8294   effect(KILL cr);
  8296   ins_cost(150);
  8297   format %{ "ADD    $dst,$src" %}
  8298   opcode(0x01);  /* Opcode 01 /r */
  8299   ins_encode( OpcP, RegMem( src, dst ) );
  8300   ins_pipe( ialu_mem_reg );
  8301 %}
  8303 // Add Memory with Immediate
  8304 instruct addI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
  8305   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
  8306   effect(KILL cr);
  8308   ins_cost(125);
  8309   format %{ "ADD    $dst,$src" %}
  8310   opcode(0x81);               /* Opcode 81 /0 id */
  8311   ins_encode( OpcSE( src ), RMopc_Mem(0x00,dst), Con8or32( src ) );
  8312   ins_pipe( ialu_mem_imm );
  8313 %}
  8315 instruct incI_mem(memory dst, immI1 src, eFlagsReg cr) %{
  8316   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
  8317   effect(KILL cr);
  8319   ins_cost(125);
  8320   format %{ "INC    $dst" %}
  8321   opcode(0xFF);               /* Opcode FF /0 */
  8322   ins_encode( OpcP, RMopc_Mem(0x00,dst));
  8323   ins_pipe( ialu_mem_imm );
  8324 %}
  8326 instruct decI_mem(memory dst, immI_M1 src, eFlagsReg cr) %{
  8327   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
  8328   effect(KILL cr);
  8330   ins_cost(125);
  8331   format %{ "DEC    $dst" %}
  8332   opcode(0xFF);               /* Opcode FF /1 */
  8333   ins_encode( OpcP, RMopc_Mem(0x01,dst));
  8334   ins_pipe( ialu_mem_imm );
  8335 %}
  8338 instruct checkCastPP( eRegP dst ) %{
  8339   match(Set dst (CheckCastPP dst));
  8341   size(0);
  8342   format %{ "#checkcastPP of $dst" %}
  8343   ins_encode( /*empty encoding*/ );
  8344   ins_pipe( empty );
  8345 %}
  8347 instruct castPP( eRegP dst ) %{
  8348   match(Set dst (CastPP dst));
  8349   format %{ "#castPP of $dst" %}
  8350   ins_encode( /*empty encoding*/ );
  8351   ins_pipe( empty );
  8352 %}
  8354 instruct castII( eRegI dst ) %{
  8355   match(Set dst (CastII dst));
  8356   format %{ "#castII of $dst" %}
  8357   ins_encode( /*empty encoding*/ );
  8358   ins_cost(0);
  8359   ins_pipe( empty );
  8360 %}
  8363 // Load-locked - same as a regular pointer load when used with compare-swap
  8364 instruct loadPLocked(eRegP dst, memory mem) %{
  8365   match(Set dst (LoadPLocked mem));
  8367   ins_cost(125);
  8368   format %{ "MOV    $dst,$mem\t# Load ptr. locked" %}
  8369   opcode(0x8B);
  8370   ins_encode( OpcP, RegMem(dst,mem));
  8371   ins_pipe( ialu_reg_mem );
  8372 %}
  8374 // LoadLong-locked - same as a volatile long load when used with compare-swap
  8375 instruct loadLLocked(stackSlotL dst, load_long_memory mem) %{
  8376   predicate(UseSSE<=1);
  8377   match(Set dst (LoadLLocked mem));
  8379   ins_cost(200);
  8380   format %{ "FILD   $mem\t# Atomic volatile long load\n\t"
  8381             "FISTp  $dst" %}
  8382   ins_encode(enc_loadL_volatile(mem,dst));
  8383   ins_pipe( fpu_reg_mem );
  8384 %}
  8386 instruct loadLX_Locked(stackSlotL dst, load_long_memory mem, regXD tmp) %{
  8387   predicate(UseSSE>=2);
  8388   match(Set dst (LoadLLocked mem));
  8389   effect(TEMP tmp);
  8390   ins_cost(180);
  8391   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
  8392             "MOVSD  $dst,$tmp" %}
  8393   ins_encode(enc_loadLX_volatile(mem, dst, tmp));
  8394   ins_pipe( pipe_slow );
  8395 %}
  8397 instruct loadLX_reg_Locked(eRegL dst, load_long_memory mem, regXD tmp) %{
  8398   predicate(UseSSE>=2);
  8399   match(Set dst (LoadLLocked mem));
  8400   effect(TEMP tmp);
  8401   ins_cost(160);
  8402   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
  8403             "MOVD   $dst.lo,$tmp\n\t"
  8404             "PSRLQ  $tmp,32\n\t"
  8405             "MOVD   $dst.hi,$tmp" %}
  8406   ins_encode(enc_loadLX_reg_volatile(mem, dst, tmp));
  8407   ins_pipe( pipe_slow );
  8408 %}
  8410 // Conditional-store of the updated heap-top.
  8411 // Used during allocation of the shared heap.
  8412 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
  8413 instruct storePConditional( memory heap_top_ptr, eAXRegP oldval, eRegP newval, eFlagsReg cr ) %{
  8414   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
  8415   // EAX is killed if there is contention, but then it's also unused.
  8416   // In the common case of no contention, EAX holds the new oop address.
  8417   format %{ "CMPXCHG $heap_top_ptr,$newval\t# If EAX==$heap_top_ptr Then store $newval into $heap_top_ptr" %}
  8418   ins_encode( lock_prefix, Opcode(0x0F), Opcode(0xB1), RegMem(newval,heap_top_ptr) );
  8419   ins_pipe( pipe_cmpxchg );
  8420 %}
  8422 // Conditional-store of an int value.
  8423 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG on Intel.
  8424 instruct storeIConditional( memory mem, eAXRegI oldval, eRegI newval, eFlagsReg cr ) %{
  8425   match(Set cr (StoreIConditional mem (Binary oldval newval)));
  8426   effect(KILL oldval);
  8427   format %{ "CMPXCHG $mem,$newval\t# If EAX==$mem Then store $newval into $mem" %}
  8428   ins_encode( lock_prefix, Opcode(0x0F), Opcode(0xB1), RegMem(newval, mem) );
  8429   ins_pipe( pipe_cmpxchg );
  8430 %}
  8432 // Conditional-store of a long value.
  8433 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG8 on Intel.
  8434 instruct storeLConditional( memory mem, eADXRegL oldval, eBCXRegL newval, eFlagsReg cr ) %{
  8435   match(Set cr (StoreLConditional mem (Binary oldval newval)));
  8436   effect(KILL oldval);
  8437   format %{ "XCHG   EBX,ECX\t# correct order for CMPXCHG8 instruction\n\t"
  8438             "CMPXCHG8 $mem,ECX:EBX\t# If EDX:EAX==$mem Then store ECX:EBX into $mem\n\t"
  8439             "XCHG   EBX,ECX"
  8440   %}
  8441   ins_encode %{
  8442     // Note: we need to swap rbx, and rcx before and after the
  8443     //       cmpxchg8 instruction because the instruction uses
  8444     //       rcx as the high order word of the new value to store but
  8445     //       our register encoding uses rbx.
  8446     __ xchgl(as_Register(EBX_enc), as_Register(ECX_enc));
  8447     if( os::is_MP() )
  8448       __ lock();
  8449     __ cmpxchg8($mem$$Address);
  8450     __ xchgl(as_Register(EBX_enc), as_Register(ECX_enc));
  8451   %}
  8452   ins_pipe( pipe_cmpxchg );
  8453 %}
  8455 // No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
  8457 instruct compareAndSwapL( eRegI res, eSIRegP mem_ptr, eADXRegL oldval, eBCXRegL newval, eFlagsReg cr ) %{
  8458   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
  8459   effect(KILL cr, KILL oldval);
  8460   format %{ "CMPXCHG8 [$mem_ptr],$newval\t# If EDX:EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
  8461             "MOV    $res,0\n\t"
  8462             "JNE,s  fail\n\t"
  8463             "MOV    $res,1\n"
  8464           "fail:" %}
  8465   ins_encode( enc_cmpxchg8(mem_ptr),
  8466               enc_flags_ne_to_boolean(res) );
  8467   ins_pipe( pipe_cmpxchg );
  8468 %}
  8470 instruct compareAndSwapP( eRegI res,  pRegP mem_ptr, eAXRegP oldval, eCXRegP newval, eFlagsReg cr) %{
  8471   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
  8472   effect(KILL cr, KILL oldval);
  8473   format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
  8474             "MOV    $res,0\n\t"
  8475             "JNE,s  fail\n\t"
  8476             "MOV    $res,1\n"
  8477           "fail:" %}
  8478   ins_encode( enc_cmpxchg(mem_ptr), enc_flags_ne_to_boolean(res) );
  8479   ins_pipe( pipe_cmpxchg );
  8480 %}
  8482 instruct compareAndSwapI( eRegI res, pRegP mem_ptr, eAXRegI oldval, eCXRegI newval, eFlagsReg cr) %{
  8483   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
  8484   effect(KILL cr, KILL oldval);
  8485   format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
  8486             "MOV    $res,0\n\t"
  8487             "JNE,s  fail\n\t"
  8488             "MOV    $res,1\n"
  8489           "fail:" %}
  8490   ins_encode( enc_cmpxchg(mem_ptr), enc_flags_ne_to_boolean(res) );
  8491   ins_pipe( pipe_cmpxchg );
  8492 %}
  8494 //----------Subtraction Instructions-------------------------------------------
  8495 // Integer Subtraction Instructions
  8496 instruct subI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
  8497   match(Set dst (SubI dst src));
  8498   effect(KILL cr);
  8500   size(2);
  8501   format %{ "SUB    $dst,$src" %}
  8502   opcode(0x2B);
  8503   ins_encode( OpcP, RegReg( dst, src) );
  8504   ins_pipe( ialu_reg_reg );
  8505 %}
  8507 instruct subI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
  8508   match(Set dst (SubI dst src));
  8509   effect(KILL cr);
  8511   format %{ "SUB    $dst,$src" %}
  8512   opcode(0x81,0x05);  /* Opcode 81 /5 */
  8513   // ins_encode( RegImm( dst, src) );
  8514   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
  8515   ins_pipe( ialu_reg );
  8516 %}
  8518 instruct subI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
  8519   match(Set dst (SubI dst (LoadI src)));
  8520   effect(KILL cr);
  8522   ins_cost(125);
  8523   format %{ "SUB    $dst,$src" %}
  8524   opcode(0x2B);
  8525   ins_encode( OpcP, RegMem( dst, src) );
  8526   ins_pipe( ialu_reg_mem );
  8527 %}
  8529 instruct subI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
  8530   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
  8531   effect(KILL cr);
  8533   ins_cost(150);
  8534   format %{ "SUB    $dst,$src" %}
  8535   opcode(0x29);  /* Opcode 29 /r */
  8536   ins_encode( OpcP, RegMem( src, dst ) );
  8537   ins_pipe( ialu_mem_reg );
  8538 %}
  8540 // Subtract from a pointer
  8541 instruct subP_eReg(eRegP dst, eRegI src, immI0 zero, eFlagsReg cr) %{
  8542   match(Set dst (AddP dst (SubI zero src)));
  8543   effect(KILL cr);
  8545   size(2);
  8546   format %{ "SUB    $dst,$src" %}
  8547   opcode(0x2B);
  8548   ins_encode( OpcP, RegReg( dst, src) );
  8549   ins_pipe( ialu_reg_reg );
  8550 %}
  8552 instruct negI_eReg(eRegI dst, immI0 zero, eFlagsReg cr) %{
  8553   match(Set dst (SubI zero dst));
  8554   effect(KILL cr);
  8556   size(2);
  8557   format %{ "NEG    $dst" %}
  8558   opcode(0xF7,0x03);  // Opcode F7 /3
  8559   ins_encode( OpcP, RegOpc( dst ) );
  8560   ins_pipe( ialu_reg );
  8561 %}
  8564 //----------Multiplication/Division Instructions-------------------------------
  8565 // Integer Multiplication Instructions
  8566 // Multiply Register
  8567 instruct mulI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
  8568   match(Set dst (MulI dst src));
  8569   effect(KILL cr);
  8571   size(3);
  8572   ins_cost(300);
  8573   format %{ "IMUL   $dst,$src" %}
  8574   opcode(0xAF, 0x0F);
  8575   ins_encode( OpcS, OpcP, RegReg( dst, src) );
  8576   ins_pipe( ialu_reg_reg_alu0 );
  8577 %}
  8579 // Multiply 32-bit Immediate
  8580 instruct mulI_eReg_imm(eRegI dst, eRegI src, immI imm, eFlagsReg cr) %{
  8581   match(Set dst (MulI src imm));
  8582   effect(KILL cr);
  8584   ins_cost(300);
  8585   format %{ "IMUL   $dst,$src,$imm" %}
  8586   opcode(0x69);  /* 69 /r id */
  8587   ins_encode( OpcSE(imm), RegReg( dst, src ), Con8or32( imm ) );
  8588   ins_pipe( ialu_reg_reg_alu0 );
  8589 %}
  8591 instruct loadConL_low_only(eADXRegL_low_only dst, immL32 src, eFlagsReg cr) %{
  8592   match(Set dst src);
  8593   effect(KILL cr);
  8595   // Note that this is artificially increased to make it more expensive than loadConL
  8596   ins_cost(250);
  8597   format %{ "MOV    EAX,$src\t// low word only" %}
  8598   opcode(0xB8);
  8599   ins_encode( LdImmL_Lo(dst, src) );
  8600   ins_pipe( ialu_reg_fat );
  8601 %}
  8603 // Multiply by 32-bit Immediate, taking the shifted high order results
  8604 //  (special case for shift by 32)
  8605 instruct mulI_imm_high(eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32 cnt, eFlagsReg cr) %{
  8606   match(Set dst (ConvL2I (RShiftL (MulL (ConvI2L src1) src2) cnt)));
  8607   predicate( _kids[0]->_kids[0]->_kids[1]->_leaf->Opcode() == Op_ConL &&
  8608              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() >= min_jint &&
  8609              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() <= max_jint );
  8610   effect(USE src1, KILL cr);
  8612   // Note that this is adjusted by 150 to compensate for the overcosting of loadConL_low_only
  8613   ins_cost(0*100 + 1*400 - 150);
  8614   format %{ "IMUL   EDX:EAX,$src1" %}
  8615   ins_encode( multiply_con_and_shift_high( dst, src1, src2, cnt, cr ) );
  8616   ins_pipe( pipe_slow );
  8617 %}
  8619 // Multiply by 32-bit Immediate, taking the shifted high order results
  8620 instruct mulI_imm_RShift_high(eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32_63 cnt, eFlagsReg cr) %{
  8621   match(Set dst (ConvL2I (RShiftL (MulL (ConvI2L src1) src2) cnt)));
  8622   predicate( _kids[0]->_kids[0]->_kids[1]->_leaf->Opcode() == Op_ConL &&
  8623              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() >= min_jint &&
  8624              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() <= max_jint );
  8625   effect(USE src1, KILL cr);
  8627   // Note that this is adjusted by 150 to compensate for the overcosting of loadConL_low_only
  8628   ins_cost(1*100 + 1*400 - 150);
  8629   format %{ "IMUL   EDX:EAX,$src1\n\t"
  8630             "SAR    EDX,$cnt-32" %}
  8631   ins_encode( multiply_con_and_shift_high( dst, src1, src2, cnt, cr ) );
  8632   ins_pipe( pipe_slow );
  8633 %}
  8635 // Multiply Memory 32-bit Immediate
  8636 instruct mulI_mem_imm(eRegI dst, memory src, immI imm, eFlagsReg cr) %{
  8637   match(Set dst (MulI (LoadI src) imm));
  8638   effect(KILL cr);
  8640   ins_cost(300);
  8641   format %{ "IMUL   $dst,$src,$imm" %}
  8642   opcode(0x69);  /* 69 /r id */
  8643   ins_encode( OpcSE(imm), RegMem( dst, src ), Con8or32( imm ) );
  8644   ins_pipe( ialu_reg_mem_alu0 );
  8645 %}
  8647 // Multiply Memory
  8648 instruct mulI(eRegI dst, memory src, eFlagsReg cr) %{
  8649   match(Set dst (MulI dst (LoadI src)));
  8650   effect(KILL cr);
  8652   ins_cost(350);
  8653   format %{ "IMUL   $dst,$src" %}
  8654   opcode(0xAF, 0x0F);
  8655   ins_encode( OpcS, OpcP, RegMem( dst, src) );
  8656   ins_pipe( ialu_reg_mem_alu0 );
  8657 %}
  8659 // Multiply Register Int to Long
  8660 instruct mulI2L(eADXRegL dst, eAXRegI src, nadxRegI src1, eFlagsReg flags) %{
  8661   // Basic Idea: long = (long)int * (long)int
  8662   match(Set dst (MulL (ConvI2L src) (ConvI2L src1)));
  8663   effect(DEF dst, USE src, USE src1, KILL flags);
  8665   ins_cost(300);
  8666   format %{ "IMUL   $dst,$src1" %}
  8668   ins_encode( long_int_multiply( dst, src1 ) );
  8669   ins_pipe( ialu_reg_reg_alu0 );
  8670 %}
  8672 instruct mulIS_eReg(eADXRegL dst, immL_32bits mask, eFlagsReg flags, eAXRegI src, nadxRegI src1) %{
  8673   // Basic Idea:  long = (int & 0xffffffffL) * (int & 0xffffffffL)
  8674   match(Set dst (MulL (AndL (ConvI2L src) mask) (AndL (ConvI2L src1) mask)));
  8675   effect(KILL flags);
  8677   ins_cost(300);
  8678   format %{ "MUL    $dst,$src1" %}
  8680   ins_encode( long_uint_multiply(dst, src1) );
  8681   ins_pipe( ialu_reg_reg_alu0 );
  8682 %}
  8684 // Multiply Register Long
  8685 instruct mulL_eReg(eADXRegL dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
  8686   match(Set dst (MulL dst src));
  8687   effect(KILL cr, TEMP tmp);
  8688   ins_cost(4*100+3*400);
  8689 // Basic idea: lo(result) = lo(x_lo * y_lo)
  8690 //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
  8691   format %{ "MOV    $tmp,$src.lo\n\t"
  8692             "IMUL   $tmp,EDX\n\t"
  8693             "MOV    EDX,$src.hi\n\t"
  8694             "IMUL   EDX,EAX\n\t"
  8695             "ADD    $tmp,EDX\n\t"
  8696             "MUL    EDX:EAX,$src.lo\n\t"
  8697             "ADD    EDX,$tmp" %}
  8698   ins_encode( long_multiply( dst, src, tmp ) );
  8699   ins_pipe( pipe_slow );
  8700 %}
  8702 // Multiply Register Long where the left operand's high 32 bits are zero
  8703 instruct mulL_eReg_lhi0(eADXRegL dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
  8704   predicate(is_operand_hi32_zero(n->in(1)));
  8705   match(Set dst (MulL dst src));
  8706   effect(KILL cr, TEMP tmp);
  8707   ins_cost(2*100+2*400);
  8708 // Basic idea: lo(result) = lo(x_lo * y_lo)
  8709 //             hi(result) = hi(x_lo * y_lo) + lo(x_lo * y_hi) where lo(x_hi * y_lo) = 0 because x_hi = 0
  8710   format %{ "MOV    $tmp,$src.hi\n\t"
  8711             "IMUL   $tmp,EAX\n\t"
  8712             "MUL    EDX:EAX,$src.lo\n\t"
  8713             "ADD    EDX,$tmp" %}
  8714   ins_encode %{
  8715     __ movl($tmp$$Register, HIGH_FROM_LOW($src$$Register));
  8716     __ imull($tmp$$Register, rax);
  8717     __ mull($src$$Register);
  8718     __ addl(rdx, $tmp$$Register);
  8719   %}
  8720   ins_pipe( pipe_slow );
  8721 %}
  8723 // Multiply Register Long where the right operand's high 32 bits are zero
  8724 instruct mulL_eReg_rhi0(eADXRegL dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
  8725   predicate(is_operand_hi32_zero(n->in(2)));
  8726   match(Set dst (MulL dst src));
  8727   effect(KILL cr, TEMP tmp);
  8728   ins_cost(2*100+2*400);
  8729 // Basic idea: lo(result) = lo(x_lo * y_lo)
  8730 //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) where lo(x_lo * y_hi) = 0 because y_hi = 0
  8731   format %{ "MOV    $tmp,$src.lo\n\t"
  8732             "IMUL   $tmp,EDX\n\t"
  8733             "MUL    EDX:EAX,$src.lo\n\t"
  8734             "ADD    EDX,$tmp" %}
  8735   ins_encode %{
  8736     __ movl($tmp$$Register, $src$$Register);
  8737     __ imull($tmp$$Register, rdx);
  8738     __ mull($src$$Register);
  8739     __ addl(rdx, $tmp$$Register);
  8740   %}
  8741   ins_pipe( pipe_slow );
  8742 %}
  8744 // Multiply Register Long where the left and the right operands' high 32 bits are zero
  8745 instruct mulL_eReg_hi0(eADXRegL dst, eRegL src, eFlagsReg cr) %{
  8746   predicate(is_operand_hi32_zero(n->in(1)) && is_operand_hi32_zero(n->in(2)));
  8747   match(Set dst (MulL dst src));
  8748   effect(KILL cr);
  8749   ins_cost(1*400);
  8750 // Basic idea: lo(result) = lo(x_lo * y_lo)
  8751 //             hi(result) = hi(x_lo * y_lo) where lo(x_hi * y_lo) = 0 and lo(x_lo * y_hi) = 0 because x_hi = 0 and y_hi = 0
  8752   format %{ "MUL    EDX:EAX,$src.lo\n\t" %}
  8753   ins_encode %{
  8754     __ mull($src$$Register);
  8755   %}
  8756   ins_pipe( pipe_slow );
  8757 %}
  8759 // Multiply Register Long by small constant
  8760 instruct mulL_eReg_con(eADXRegL dst, immL_127 src, eRegI tmp, eFlagsReg cr) %{
  8761   match(Set dst (MulL dst src));
  8762   effect(KILL cr, TEMP tmp);
  8763   ins_cost(2*100+2*400);
  8764   size(12);
  8765 // Basic idea: lo(result) = lo(src * EAX)
  8766 //             hi(result) = hi(src * EAX) + lo(src * EDX)
  8767   format %{ "IMUL   $tmp,EDX,$src\n\t"
  8768             "MOV    EDX,$src\n\t"
  8769             "MUL    EDX\t# EDX*EAX -> EDX:EAX\n\t"
  8770             "ADD    EDX,$tmp" %}
  8771   ins_encode( long_multiply_con( dst, src, tmp ) );
  8772   ins_pipe( pipe_slow );
  8773 %}
  8775 // Integer DIV with Register
  8776 instruct divI_eReg(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{
  8777   match(Set rax (DivI rax div));
  8778   effect(KILL rdx, KILL cr);
  8779   size(26);
  8780   ins_cost(30*100+10*100);
  8781   format %{ "CMP    EAX,0x80000000\n\t"
  8782             "JNE,s  normal\n\t"
  8783             "XOR    EDX,EDX\n\t"
  8784             "CMP    ECX,-1\n\t"
  8785             "JE,s   done\n"
  8786     "normal: CDQ\n\t"
  8787             "IDIV   $div\n\t"
  8788     "done:"        %}
  8789   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
  8790   ins_encode( cdq_enc, OpcP, RegOpc(div) );
  8791   ins_pipe( ialu_reg_reg_alu0 );
  8792 %}
  8794 // Divide Register Long
  8795 instruct divL_eReg( eADXRegL dst, eRegL src1, eRegL src2, eFlagsReg cr, eCXRegI cx, eBXRegI bx ) %{
  8796   match(Set dst (DivL src1 src2));
  8797   effect( KILL cr, KILL cx, KILL bx );
  8798   ins_cost(10000);
  8799   format %{ "PUSH   $src1.hi\n\t"
  8800             "PUSH   $src1.lo\n\t"
  8801             "PUSH   $src2.hi\n\t"
  8802             "PUSH   $src2.lo\n\t"
  8803             "CALL   SharedRuntime::ldiv\n\t"
  8804             "ADD    ESP,16" %}
  8805   ins_encode( long_div(src1,src2) );
  8806   ins_pipe( pipe_slow );
  8807 %}
  8809 // Integer DIVMOD with Register, both quotient and mod results
  8810 instruct divModI_eReg_divmod(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{
  8811   match(DivModI rax div);
  8812   effect(KILL cr);
  8813   size(26);
  8814   ins_cost(30*100+10*100);
  8815   format %{ "CMP    EAX,0x80000000\n\t"
  8816             "JNE,s  normal\n\t"
  8817             "XOR    EDX,EDX\n\t"
  8818             "CMP    ECX,-1\n\t"
  8819             "JE,s   done\n"
  8820     "normal: CDQ\n\t"
  8821             "IDIV   $div\n\t"
  8822     "done:"        %}
  8823   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
  8824   ins_encode( cdq_enc, OpcP, RegOpc(div) );
  8825   ins_pipe( pipe_slow );
  8826 %}
  8828 // Integer MOD with Register
  8829 instruct modI_eReg(eDXRegI rdx, eAXRegI rax, eCXRegI div, eFlagsReg cr) %{
  8830   match(Set rdx (ModI rax div));
  8831   effect(KILL rax, KILL cr);
  8833   size(26);
  8834   ins_cost(300);
  8835   format %{ "CDQ\n\t"
  8836             "IDIV   $div" %}
  8837   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
  8838   ins_encode( cdq_enc, OpcP, RegOpc(div) );
  8839   ins_pipe( ialu_reg_reg_alu0 );
  8840 %}
  8842 // Remainder Register Long
  8843 instruct modL_eReg( eADXRegL dst, eRegL src1, eRegL src2, eFlagsReg cr, eCXRegI cx, eBXRegI bx ) %{
  8844   match(Set dst (ModL src1 src2));
  8845   effect( KILL cr, KILL cx, KILL bx );
  8846   ins_cost(10000);
  8847   format %{ "PUSH   $src1.hi\n\t"
  8848             "PUSH   $src1.lo\n\t"
  8849             "PUSH   $src2.hi\n\t"
  8850             "PUSH   $src2.lo\n\t"
  8851             "CALL   SharedRuntime::lrem\n\t"
  8852             "ADD    ESP,16" %}
  8853   ins_encode( long_mod(src1,src2) );
  8854   ins_pipe( pipe_slow );
  8855 %}
  8857 // Divide Register Long (no special case since divisor != -1)
  8858 instruct divL_eReg_imm32( eADXRegL dst, immL32 imm, eRegI tmp, eRegI tmp2, eFlagsReg cr ) %{
  8859   match(Set dst (DivL dst imm));
  8860   effect( TEMP tmp, TEMP tmp2, KILL cr );
  8861   ins_cost(1000);
  8862   format %{ "MOV    $tmp,abs($imm) # ldiv EDX:EAX,$imm\n\t"
  8863             "XOR    $tmp2,$tmp2\n\t"
  8864             "CMP    $tmp,EDX\n\t"
  8865             "JA,s   fast\n\t"
  8866             "MOV    $tmp2,EAX\n\t"
  8867             "MOV    EAX,EDX\n\t"
  8868             "MOV    EDX,0\n\t"
  8869             "JLE,s  pos\n\t"
  8870             "LNEG   EAX : $tmp2\n\t"
  8871             "DIV    $tmp # unsigned division\n\t"
  8872             "XCHG   EAX,$tmp2\n\t"
  8873             "DIV    $tmp\n\t"
  8874             "LNEG   $tmp2 : EAX\n\t"
  8875             "JMP,s  done\n"
  8876     "pos:\n\t"
  8877             "DIV    $tmp\n\t"
  8878             "XCHG   EAX,$tmp2\n"
  8879     "fast:\n\t"
  8880             "DIV    $tmp\n"
  8881     "done:\n\t"
  8882             "MOV    EDX,$tmp2\n\t"
  8883             "NEG    EDX:EAX # if $imm < 0" %}
  8884   ins_encode %{
  8885     int con = (int)$imm$$constant;
  8886     assert(con != 0 && con != -1 && con != min_jint, "wrong divisor");
  8887     int pcon = (con > 0) ? con : -con;
  8888     Label Lfast, Lpos, Ldone;
  8890     __ movl($tmp$$Register, pcon);
  8891     __ xorl($tmp2$$Register,$tmp2$$Register);
  8892     __ cmpl($tmp$$Register, HIGH_FROM_LOW($dst$$Register));
  8893     __ jccb(Assembler::above, Lfast); // result fits into 32 bit
  8895     __ movl($tmp2$$Register, $dst$$Register); // save
  8896     __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
  8897     __ movl(HIGH_FROM_LOW($dst$$Register),0); // preserve flags
  8898     __ jccb(Assembler::lessEqual, Lpos); // result is positive
  8900     // Negative dividend.
  8901     // convert value to positive to use unsigned division
  8902     __ lneg($dst$$Register, $tmp2$$Register);
  8903     __ divl($tmp$$Register);
  8904     __ xchgl($dst$$Register, $tmp2$$Register);
  8905     __ divl($tmp$$Register);
  8906     // revert result back to negative
  8907     __ lneg($tmp2$$Register, $dst$$Register);
  8908     __ jmpb(Ldone);
  8910     __ bind(Lpos);
  8911     __ divl($tmp$$Register); // Use unsigned division
  8912     __ xchgl($dst$$Register, $tmp2$$Register);
  8913     // Fallthrow for final divide, tmp2 has 32 bit hi result
  8915     __ bind(Lfast);
  8916     // fast path: src is positive
  8917     __ divl($tmp$$Register); // Use unsigned division
  8919     __ bind(Ldone);
  8920     __ movl(HIGH_FROM_LOW($dst$$Register),$tmp2$$Register);
  8921     if (con < 0) {
  8922       __ lneg(HIGH_FROM_LOW($dst$$Register), $dst$$Register);
  8924   %}
  8925   ins_pipe( pipe_slow );
  8926 %}
  8928 // Remainder Register Long (remainder fit into 32 bits)
  8929 instruct modL_eReg_imm32( eADXRegL dst, immL32 imm, eRegI tmp, eRegI tmp2, eFlagsReg cr ) %{
  8930   match(Set dst (ModL dst imm));
  8931   effect( TEMP tmp, TEMP tmp2, KILL cr );
  8932   ins_cost(1000);
  8933   format %{ "MOV    $tmp,abs($imm) # lrem EDX:EAX,$imm\n\t"
  8934             "CMP    $tmp,EDX\n\t"
  8935             "JA,s   fast\n\t"
  8936             "MOV    $tmp2,EAX\n\t"
  8937             "MOV    EAX,EDX\n\t"
  8938             "MOV    EDX,0\n\t"
  8939             "JLE,s  pos\n\t"
  8940             "LNEG   EAX : $tmp2\n\t"
  8941             "DIV    $tmp # unsigned division\n\t"
  8942             "MOV    EAX,$tmp2\n\t"
  8943             "DIV    $tmp\n\t"
  8944             "NEG    EDX\n\t"
  8945             "JMP,s  done\n"
  8946     "pos:\n\t"
  8947             "DIV    $tmp\n\t"
  8948             "MOV    EAX,$tmp2\n"
  8949     "fast:\n\t"
  8950             "DIV    $tmp\n"
  8951     "done:\n\t"
  8952             "MOV    EAX,EDX\n\t"
  8953             "SAR    EDX,31\n\t" %}
  8954   ins_encode %{
  8955     int con = (int)$imm$$constant;
  8956     assert(con != 0 && con != -1 && con != min_jint, "wrong divisor");
  8957     int pcon = (con > 0) ? con : -con;
  8958     Label  Lfast, Lpos, Ldone;
  8960     __ movl($tmp$$Register, pcon);
  8961     __ cmpl($tmp$$Register, HIGH_FROM_LOW($dst$$Register));
  8962     __ jccb(Assembler::above, Lfast); // src is positive and result fits into 32 bit
  8964     __ movl($tmp2$$Register, $dst$$Register); // save
  8965     __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
  8966     __ movl(HIGH_FROM_LOW($dst$$Register),0); // preserve flags
  8967     __ jccb(Assembler::lessEqual, Lpos); // result is positive
  8969     // Negative dividend.
  8970     // convert value to positive to use unsigned division
  8971     __ lneg($dst$$Register, $tmp2$$Register);
  8972     __ divl($tmp$$Register);
  8973     __ movl($dst$$Register, $tmp2$$Register);
  8974     __ divl($tmp$$Register);
  8975     // revert remainder back to negative
  8976     __ negl(HIGH_FROM_LOW($dst$$Register));
  8977     __ jmpb(Ldone);
  8979     __ bind(Lpos);
  8980     __ divl($tmp$$Register);
  8981     __ movl($dst$$Register, $tmp2$$Register);
  8983     __ bind(Lfast);
  8984     // fast path: src is positive
  8985     __ divl($tmp$$Register);
  8987     __ bind(Ldone);
  8988     __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
  8989     __ sarl(HIGH_FROM_LOW($dst$$Register), 31); // result sign
  8991   %}
  8992   ins_pipe( pipe_slow );
  8993 %}
  8995 // Integer Shift Instructions
  8996 // Shift Left by one
  8997 instruct shlI_eReg_1(eRegI dst, immI1 shift, eFlagsReg cr) %{
  8998   match(Set dst (LShiftI dst shift));
  8999   effect(KILL cr);
  9001   size(2);
  9002   format %{ "SHL    $dst,$shift" %}
  9003   opcode(0xD1, 0x4);  /* D1 /4 */
  9004   ins_encode( OpcP, RegOpc( dst ) );
  9005   ins_pipe( ialu_reg );
  9006 %}
  9008 // Shift Left by 8-bit immediate
  9009 instruct salI_eReg_imm(eRegI dst, immI8 shift, eFlagsReg cr) %{
  9010   match(Set dst (LShiftI dst shift));
  9011   effect(KILL cr);
  9013   size(3);
  9014   format %{ "SHL    $dst,$shift" %}
  9015   opcode(0xC1, 0x4);  /* C1 /4 ib */
  9016   ins_encode( RegOpcImm( dst, shift) );
  9017   ins_pipe( ialu_reg );
  9018 %}
  9020 // Shift Left by variable
  9021 instruct salI_eReg_CL(eRegI dst, eCXRegI shift, eFlagsReg cr) %{
  9022   match(Set dst (LShiftI dst shift));
  9023   effect(KILL cr);
  9025   size(2);
  9026   format %{ "SHL    $dst,$shift" %}
  9027   opcode(0xD3, 0x4);  /* D3 /4 */
  9028   ins_encode( OpcP, RegOpc( dst ) );
  9029   ins_pipe( ialu_reg_reg );
  9030 %}
  9032 // Arithmetic shift right by one
  9033 instruct sarI_eReg_1(eRegI dst, immI1 shift, eFlagsReg cr) %{
  9034   match(Set dst (RShiftI dst shift));
  9035   effect(KILL cr);
  9037   size(2);
  9038   format %{ "SAR    $dst,$shift" %}
  9039   opcode(0xD1, 0x7);  /* D1 /7 */
  9040   ins_encode( OpcP, RegOpc( dst ) );
  9041   ins_pipe( ialu_reg );
  9042 %}
  9044 // Arithmetic shift right by one
  9045 instruct sarI_mem_1(memory dst, immI1 shift, eFlagsReg cr) %{
  9046   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
  9047   effect(KILL cr);
  9048   format %{ "SAR    $dst,$shift" %}
  9049   opcode(0xD1, 0x7);  /* D1 /7 */
  9050   ins_encode( OpcP, RMopc_Mem(secondary,dst) );
  9051   ins_pipe( ialu_mem_imm );
  9052 %}
  9054 // Arithmetic Shift Right by 8-bit immediate
  9055 instruct sarI_eReg_imm(eRegI dst, immI8 shift, eFlagsReg cr) %{
  9056   match(Set dst (RShiftI dst shift));
  9057   effect(KILL cr);
  9059   size(3);
  9060   format %{ "SAR    $dst,$shift" %}
  9061   opcode(0xC1, 0x7);  /* C1 /7 ib */
  9062   ins_encode( RegOpcImm( dst, shift ) );
  9063   ins_pipe( ialu_mem_imm );
  9064 %}
  9066 // Arithmetic Shift Right by 8-bit immediate
  9067 instruct sarI_mem_imm(memory dst, immI8 shift, eFlagsReg cr) %{
  9068   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
  9069   effect(KILL cr);
  9071   format %{ "SAR    $dst,$shift" %}
  9072   opcode(0xC1, 0x7);  /* C1 /7 ib */
  9073   ins_encode( OpcP, RMopc_Mem(secondary, dst ), Con8or32( shift ) );
  9074   ins_pipe( ialu_mem_imm );
  9075 %}
  9077 // Arithmetic Shift Right by variable
  9078 instruct sarI_eReg_CL(eRegI dst, eCXRegI shift, eFlagsReg cr) %{
  9079   match(Set dst (RShiftI dst shift));
  9080   effect(KILL cr);
  9082   size(2);
  9083   format %{ "SAR    $dst,$shift" %}
  9084   opcode(0xD3, 0x7);  /* D3 /7 */
  9085   ins_encode( OpcP, RegOpc( dst ) );
  9086   ins_pipe( ialu_reg_reg );
  9087 %}
  9089 // Logical shift right by one
  9090 instruct shrI_eReg_1(eRegI dst, immI1 shift, eFlagsReg cr) %{
  9091   match(Set dst (URShiftI dst shift));
  9092   effect(KILL cr);
  9094   size(2);
  9095   format %{ "SHR    $dst,$shift" %}
  9096   opcode(0xD1, 0x5);  /* D1 /5 */
  9097   ins_encode( OpcP, RegOpc( dst ) );
  9098   ins_pipe( ialu_reg );
  9099 %}
  9101 // Logical Shift Right by 8-bit immediate
  9102 instruct shrI_eReg_imm(eRegI dst, immI8 shift, eFlagsReg cr) %{
  9103   match(Set dst (URShiftI dst shift));
  9104   effect(KILL cr);
  9106   size(3);
  9107   format %{ "SHR    $dst,$shift" %}
  9108   opcode(0xC1, 0x5);  /* C1 /5 ib */
  9109   ins_encode( RegOpcImm( dst, shift) );
  9110   ins_pipe( ialu_reg );
  9111 %}
  9114 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
  9115 // This idiom is used by the compiler for the i2b bytecode.
  9116 instruct i2b(eRegI dst, xRegI src, immI_24 twentyfour) %{
  9117   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
  9119   size(3);
  9120   format %{ "MOVSX  $dst,$src :8" %}
  9121   ins_encode %{
  9122     __ movsbl($dst$$Register, $src$$Register);
  9123   %}
  9124   ins_pipe(ialu_reg_reg);
  9125 %}
  9127 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
  9128 // This idiom is used by the compiler the i2s bytecode.
  9129 instruct i2s(eRegI dst, xRegI src, immI_16 sixteen) %{
  9130   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
  9132   size(3);
  9133   format %{ "MOVSX  $dst,$src :16" %}
  9134   ins_encode %{
  9135     __ movswl($dst$$Register, $src$$Register);
  9136   %}
  9137   ins_pipe(ialu_reg_reg);
  9138 %}
  9141 // Logical Shift Right by variable
  9142 instruct shrI_eReg_CL(eRegI dst, eCXRegI shift, eFlagsReg cr) %{
  9143   match(Set dst (URShiftI dst shift));
  9144   effect(KILL cr);
  9146   size(2);
  9147   format %{ "SHR    $dst,$shift" %}
  9148   opcode(0xD3, 0x5);  /* D3 /5 */
  9149   ins_encode( OpcP, RegOpc( dst ) );
  9150   ins_pipe( ialu_reg_reg );
  9151 %}
  9154 //----------Logical Instructions-----------------------------------------------
  9155 //----------Integer Logical Instructions---------------------------------------
  9156 // And Instructions
  9157 // And Register with Register
  9158 instruct andI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
  9159   match(Set dst (AndI dst src));
  9160   effect(KILL cr);
  9162   size(2);
  9163   format %{ "AND    $dst,$src" %}
  9164   opcode(0x23);
  9165   ins_encode( OpcP, RegReg( dst, src) );
  9166   ins_pipe( ialu_reg_reg );
  9167 %}
  9169 // And Register with Immediate
  9170 instruct andI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
  9171   match(Set dst (AndI dst src));
  9172   effect(KILL cr);
  9174   format %{ "AND    $dst,$src" %}
  9175   opcode(0x81,0x04);  /* Opcode 81 /4 */
  9176   // ins_encode( RegImm( dst, src) );
  9177   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
  9178   ins_pipe( ialu_reg );
  9179 %}
  9181 // And Register with Memory
  9182 instruct andI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
  9183   match(Set dst (AndI dst (LoadI src)));
  9184   effect(KILL cr);
  9186   ins_cost(125);
  9187   format %{ "AND    $dst,$src" %}
  9188   opcode(0x23);
  9189   ins_encode( OpcP, RegMem( dst, src) );
  9190   ins_pipe( ialu_reg_mem );
  9191 %}
  9193 // And Memory with Register
  9194 instruct andI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
  9195   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
  9196   effect(KILL cr);
  9198   ins_cost(150);
  9199   format %{ "AND    $dst,$src" %}
  9200   opcode(0x21);  /* Opcode 21 /r */
  9201   ins_encode( OpcP, RegMem( src, dst ) );
  9202   ins_pipe( ialu_mem_reg );
  9203 %}
  9205 // And Memory with Immediate
  9206 instruct andI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
  9207   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
  9208   effect(KILL cr);
  9210   ins_cost(125);
  9211   format %{ "AND    $dst,$src" %}
  9212   opcode(0x81, 0x4);  /* Opcode 81 /4 id */
  9213   // ins_encode( MemImm( dst, src) );
  9214   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
  9215   ins_pipe( ialu_mem_imm );
  9216 %}
  9218 // Or Instructions
  9219 // Or Register with Register
  9220 instruct orI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
  9221   match(Set dst (OrI dst src));
  9222   effect(KILL cr);
  9224   size(2);
  9225   format %{ "OR     $dst,$src" %}
  9226   opcode(0x0B);
  9227   ins_encode( OpcP, RegReg( dst, src) );
  9228   ins_pipe( ialu_reg_reg );
  9229 %}
  9231 instruct orI_eReg_castP2X(eRegI dst, eRegP src, eFlagsReg cr) %{
  9232   match(Set dst (OrI dst (CastP2X src)));
  9233   effect(KILL cr);
  9235   size(2);
  9236   format %{ "OR     $dst,$src" %}
  9237   opcode(0x0B);
  9238   ins_encode( OpcP, RegReg( dst, src) );
  9239   ins_pipe( ialu_reg_reg );
  9240 %}
  9243 // Or Register with Immediate
  9244 instruct orI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
  9245   match(Set dst (OrI dst src));
  9246   effect(KILL cr);
  9248   format %{ "OR     $dst,$src" %}
  9249   opcode(0x81,0x01);  /* Opcode 81 /1 id */
  9250   // ins_encode( RegImm( dst, src) );
  9251   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
  9252   ins_pipe( ialu_reg );
  9253 %}
  9255 // Or Register with Memory
  9256 instruct orI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
  9257   match(Set dst (OrI dst (LoadI src)));
  9258   effect(KILL cr);
  9260   ins_cost(125);
  9261   format %{ "OR     $dst,$src" %}
  9262   opcode(0x0B);
  9263   ins_encode( OpcP, RegMem( dst, src) );
  9264   ins_pipe( ialu_reg_mem );
  9265 %}
  9267 // Or Memory with Register
  9268 instruct orI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
  9269   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
  9270   effect(KILL cr);
  9272   ins_cost(150);
  9273   format %{ "OR     $dst,$src" %}
  9274   opcode(0x09);  /* Opcode 09 /r */
  9275   ins_encode( OpcP, RegMem( src, dst ) );
  9276   ins_pipe( ialu_mem_reg );
  9277 %}
  9279 // Or Memory with Immediate
  9280 instruct orI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
  9281   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
  9282   effect(KILL cr);
  9284   ins_cost(125);
  9285   format %{ "OR     $dst,$src" %}
  9286   opcode(0x81,0x1);  /* Opcode 81 /1 id */
  9287   // ins_encode( MemImm( dst, src) );
  9288   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
  9289   ins_pipe( ialu_mem_imm );
  9290 %}
  9292 // ROL/ROR
  9293 // ROL expand
  9294 instruct rolI_eReg_imm1(eRegI dst, immI1 shift, eFlagsReg cr) %{
  9295   effect(USE_DEF dst, USE shift, KILL cr);
  9297   format %{ "ROL    $dst, $shift" %}
  9298   opcode(0xD1, 0x0); /* Opcode D1 /0 */
  9299   ins_encode( OpcP, RegOpc( dst ));
  9300   ins_pipe( ialu_reg );
  9301 %}
  9303 instruct rolI_eReg_imm8(eRegI dst, immI8 shift, eFlagsReg cr) %{
  9304   effect(USE_DEF dst, USE shift, KILL cr);
  9306   format %{ "ROL    $dst, $shift" %}
  9307   opcode(0xC1, 0x0); /*Opcode /C1  /0  */
  9308   ins_encode( RegOpcImm(dst, shift) );
  9309   ins_pipe(ialu_reg);
  9310 %}
  9312 instruct rolI_eReg_CL(ncxRegI dst, eCXRegI shift, eFlagsReg cr) %{
  9313   effect(USE_DEF dst, USE shift, KILL cr);
  9315   format %{ "ROL    $dst, $shift" %}
  9316   opcode(0xD3, 0x0);    /* Opcode D3 /0 */
  9317   ins_encode(OpcP, RegOpc(dst));
  9318   ins_pipe( ialu_reg_reg );
  9319 %}
  9320 // end of ROL expand
  9322 // ROL 32bit by one once
  9323 instruct rolI_eReg_i1(eRegI dst, immI1 lshift, immI_M1 rshift, eFlagsReg cr) %{
  9324   match(Set dst ( OrI (LShiftI dst lshift) (URShiftI dst rshift)));
  9326   expand %{
  9327     rolI_eReg_imm1(dst, lshift, cr);
  9328   %}
  9329 %}
  9331 // ROL 32bit var by imm8 once
  9332 instruct rolI_eReg_i8(eRegI dst, immI8 lshift, immI8 rshift, eFlagsReg cr) %{
  9333   predicate(  0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
  9334   match(Set dst ( OrI (LShiftI dst lshift) (URShiftI dst rshift)));
  9336   expand %{
  9337     rolI_eReg_imm8(dst, lshift, cr);
  9338   %}
  9339 %}
  9341 // ROL 32bit var by var once
  9342 instruct rolI_eReg_Var_C0(ncxRegI dst, eCXRegI shift, immI0 zero, eFlagsReg cr) %{
  9343   match(Set dst ( OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
  9345   expand %{
  9346     rolI_eReg_CL(dst, shift, cr);
  9347   %}
  9348 %}
  9350 // ROL 32bit var by var once
  9351 instruct rolI_eReg_Var_C32(ncxRegI dst, eCXRegI shift, immI_32 c32, eFlagsReg cr) %{
  9352   match(Set dst ( OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
  9354   expand %{
  9355     rolI_eReg_CL(dst, shift, cr);
  9356   %}
  9357 %}
  9359 // ROR expand
  9360 instruct rorI_eReg_imm1(eRegI dst, immI1 shift, eFlagsReg cr) %{
  9361   effect(USE_DEF dst, USE shift, KILL cr);
  9363   format %{ "ROR    $dst, $shift" %}
  9364   opcode(0xD1,0x1);  /* Opcode D1 /1 */
  9365   ins_encode( OpcP, RegOpc( dst ) );
  9366   ins_pipe( ialu_reg );
  9367 %}
  9369 instruct rorI_eReg_imm8(eRegI dst, immI8 shift, eFlagsReg cr) %{
  9370   effect (USE_DEF dst, USE shift, KILL cr);
  9372   format %{ "ROR    $dst, $shift" %}
  9373   opcode(0xC1, 0x1); /* Opcode /C1 /1 ib */
  9374   ins_encode( RegOpcImm(dst, shift) );
  9375   ins_pipe( ialu_reg );
  9376 %}
  9378 instruct rorI_eReg_CL(ncxRegI dst, eCXRegI shift, eFlagsReg cr)%{
  9379   effect(USE_DEF dst, USE shift, KILL cr);
  9381   format %{ "ROR    $dst, $shift" %}
  9382   opcode(0xD3, 0x1);    /* Opcode D3 /1 */
  9383   ins_encode(OpcP, RegOpc(dst));
  9384   ins_pipe( ialu_reg_reg );
  9385 %}
  9386 // end of ROR expand
  9388 // ROR right once
  9389 instruct rorI_eReg_i1(eRegI dst, immI1 rshift, immI_M1 lshift, eFlagsReg cr) %{
  9390   match(Set dst ( OrI (URShiftI dst rshift) (LShiftI dst lshift)));
  9392   expand %{
  9393     rorI_eReg_imm1(dst, rshift, cr);
  9394   %}
  9395 %}
  9397 // ROR 32bit by immI8 once
  9398 instruct rorI_eReg_i8(eRegI dst, immI8 rshift, immI8 lshift, eFlagsReg cr) %{
  9399   predicate(  0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
  9400   match(Set dst ( OrI (URShiftI dst rshift) (LShiftI dst lshift)));
  9402   expand %{
  9403     rorI_eReg_imm8(dst, rshift, cr);
  9404   %}
  9405 %}
  9407 // ROR 32bit var by var once
  9408 instruct rorI_eReg_Var_C0(ncxRegI dst, eCXRegI shift, immI0 zero, eFlagsReg cr) %{
  9409   match(Set dst ( OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
  9411   expand %{
  9412     rorI_eReg_CL(dst, shift, cr);
  9413   %}
  9414 %}
  9416 // ROR 32bit var by var once
  9417 instruct rorI_eReg_Var_C32(ncxRegI dst, eCXRegI shift, immI_32 c32, eFlagsReg cr) %{
  9418   match(Set dst ( OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
  9420   expand %{
  9421     rorI_eReg_CL(dst, shift, cr);
  9422   %}
  9423 %}
  9425 // Xor Instructions
  9426 // Xor Register with Register
  9427 instruct xorI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
  9428   match(Set dst (XorI dst src));
  9429   effect(KILL cr);
  9431   size(2);
  9432   format %{ "XOR    $dst,$src" %}
  9433   opcode(0x33);
  9434   ins_encode( OpcP, RegReg( dst, src) );
  9435   ins_pipe( ialu_reg_reg );
  9436 %}
  9438 // Xor Register with Immediate -1
  9439 instruct xorI_eReg_im1(eRegI dst, immI_M1 imm) %{
  9440   match(Set dst (XorI dst imm));  
  9442   size(2);
  9443   format %{ "NOT    $dst" %}  
  9444   ins_encode %{
  9445      __ notl($dst$$Register);
  9446   %}
  9447   ins_pipe( ialu_reg );
  9448 %}
  9450 // Xor Register with Immediate
  9451 instruct xorI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
  9452   match(Set dst (XorI dst src));
  9453   effect(KILL cr);
  9455   format %{ "XOR    $dst,$src" %}
  9456   opcode(0x81,0x06);  /* Opcode 81 /6 id */
  9457   // ins_encode( RegImm( dst, src) );
  9458   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
  9459   ins_pipe( ialu_reg );
  9460 %}
  9462 // Xor Register with Memory
  9463 instruct xorI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
  9464   match(Set dst (XorI dst (LoadI src)));
  9465   effect(KILL cr);
  9467   ins_cost(125);
  9468   format %{ "XOR    $dst,$src" %}
  9469   opcode(0x33);
  9470   ins_encode( OpcP, RegMem(dst, src) );
  9471   ins_pipe( ialu_reg_mem );
  9472 %}
  9474 // Xor Memory with Register
  9475 instruct xorI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
  9476   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
  9477   effect(KILL cr);
  9479   ins_cost(150);
  9480   format %{ "XOR    $dst,$src" %}
  9481   opcode(0x31);  /* Opcode 31 /r */
  9482   ins_encode( OpcP, RegMem( src, dst ) );
  9483   ins_pipe( ialu_mem_reg );
  9484 %}
  9486 // Xor Memory with Immediate
  9487 instruct xorI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
  9488   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
  9489   effect(KILL cr);
  9491   ins_cost(125);
  9492   format %{ "XOR    $dst,$src" %}
  9493   opcode(0x81,0x6);  /* Opcode 81 /6 id */
  9494   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
  9495   ins_pipe( ialu_mem_imm );
  9496 %}
  9498 //----------Convert Int to Boolean---------------------------------------------
  9500 instruct movI_nocopy(eRegI dst, eRegI src) %{
  9501   effect( DEF dst, USE src );
  9502   format %{ "MOV    $dst,$src" %}
  9503   ins_encode( enc_Copy( dst, src) );
  9504   ins_pipe( ialu_reg_reg );
  9505 %}
  9507 instruct ci2b( eRegI dst, eRegI src, eFlagsReg cr ) %{
  9508   effect( USE_DEF dst, USE src, KILL cr );
  9510   size(4);
  9511   format %{ "NEG    $dst\n\t"
  9512             "ADC    $dst,$src" %}
  9513   ins_encode( neg_reg(dst),
  9514               OpcRegReg(0x13,dst,src) );
  9515   ins_pipe( ialu_reg_reg_long );
  9516 %}
  9518 instruct convI2B( eRegI dst, eRegI src, eFlagsReg cr ) %{
  9519   match(Set dst (Conv2B src));
  9521   expand %{
  9522     movI_nocopy(dst,src);
  9523     ci2b(dst,src,cr);
  9524   %}
  9525 %}
  9527 instruct movP_nocopy(eRegI dst, eRegP src) %{
  9528   effect( DEF dst, USE src );
  9529   format %{ "MOV    $dst,$src" %}
  9530   ins_encode( enc_Copy( dst, src) );
  9531   ins_pipe( ialu_reg_reg );
  9532 %}
  9534 instruct cp2b( eRegI dst, eRegP src, eFlagsReg cr ) %{
  9535   effect( USE_DEF dst, USE src, KILL cr );
  9536   format %{ "NEG    $dst\n\t"
  9537             "ADC    $dst,$src" %}
  9538   ins_encode( neg_reg(dst),
  9539               OpcRegReg(0x13,dst,src) );
  9540   ins_pipe( ialu_reg_reg_long );
  9541 %}
  9543 instruct convP2B( eRegI dst, eRegP src, eFlagsReg cr ) %{
  9544   match(Set dst (Conv2B src));
  9546   expand %{
  9547     movP_nocopy(dst,src);
  9548     cp2b(dst,src,cr);
  9549   %}
  9550 %}
  9552 instruct cmpLTMask( eCXRegI dst, ncxRegI p, ncxRegI q, eFlagsReg cr ) %{
  9553   match(Set dst (CmpLTMask p q));
  9554   effect( KILL cr );
  9555   ins_cost(400);
  9557   // SETlt can only use low byte of EAX,EBX, ECX, or EDX as destination
  9558   format %{ "XOR    $dst,$dst\n\t"
  9559             "CMP    $p,$q\n\t"
  9560             "SETlt  $dst\n\t"
  9561             "NEG    $dst" %}
  9562   ins_encode( OpcRegReg(0x33,dst,dst),
  9563               OpcRegReg(0x3B,p,q),
  9564               setLT_reg(dst), neg_reg(dst) );
  9565   ins_pipe( pipe_slow );
  9566 %}
  9568 instruct cmpLTMask0( eRegI dst, immI0 zero, eFlagsReg cr ) %{
  9569   match(Set dst (CmpLTMask dst zero));
  9570   effect( DEF dst, KILL cr );
  9571   ins_cost(100);
  9573   format %{ "SAR    $dst,31" %}
  9574   opcode(0xC1, 0x7);  /* C1 /7 ib */
  9575   ins_encode( RegOpcImm( dst, 0x1F ) );
  9576   ins_pipe( ialu_reg );
  9577 %}
  9580 instruct cadd_cmpLTMask( ncxRegI p, ncxRegI q, ncxRegI y, eCXRegI tmp, eFlagsReg cr ) %{
  9581   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
  9582   effect( KILL tmp, KILL cr );
  9583   ins_cost(400);
  9584   // annoyingly, $tmp has no edges so you cant ask for it in
  9585   // any format or encoding
  9586   format %{ "SUB    $p,$q\n\t"
  9587             "SBB    ECX,ECX\n\t"
  9588             "AND    ECX,$y\n\t"
  9589             "ADD    $p,ECX" %}
  9590   ins_encode( enc_cmpLTP(p,q,y,tmp) );
  9591   ins_pipe( pipe_cmplt );
  9592 %}
  9594 /* If I enable this, I encourage spilling in the inner loop of compress.
  9595 instruct cadd_cmpLTMask_mem( ncxRegI p, ncxRegI q, memory y, eCXRegI tmp, eFlagsReg cr ) %{
  9596   match(Set p (AddI (AndI (CmpLTMask p q) (LoadI y)) (SubI p q)));
  9597   effect( USE_KILL tmp, KILL cr );
  9598   ins_cost(400);
  9600   format %{ "SUB    $p,$q\n\t"
  9601             "SBB    ECX,ECX\n\t"
  9602             "AND    ECX,$y\n\t"
  9603             "ADD    $p,ECX" %}
  9604   ins_encode( enc_cmpLTP_mem(p,q,y,tmp) );
  9605 %}
  9606 */
  9608 //----------Long Instructions------------------------------------------------
  9609 // Add Long Register with Register
  9610 instruct addL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
  9611   match(Set dst (AddL dst src));
  9612   effect(KILL cr);
  9613   ins_cost(200);
  9614   format %{ "ADD    $dst.lo,$src.lo\n\t"
  9615             "ADC    $dst.hi,$src.hi" %}
  9616   opcode(0x03, 0x13);
  9617   ins_encode( RegReg_Lo(dst, src), RegReg_Hi(dst,src) );
  9618   ins_pipe( ialu_reg_reg_long );
  9619 %}
  9621 // Add Long Register with Immediate
  9622 instruct addL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
  9623   match(Set dst (AddL dst src));
  9624   effect(KILL cr);
  9625   format %{ "ADD    $dst.lo,$src.lo\n\t"
  9626             "ADC    $dst.hi,$src.hi" %}
  9627   opcode(0x81,0x00,0x02);  /* Opcode 81 /0, 81 /2 */
  9628   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
  9629   ins_pipe( ialu_reg_long );
  9630 %}
  9632 // Add Long Register with Memory
  9633 instruct addL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
  9634   match(Set dst (AddL dst (LoadL mem)));
  9635   effect(KILL cr);
  9636   ins_cost(125);
  9637   format %{ "ADD    $dst.lo,$mem\n\t"
  9638             "ADC    $dst.hi,$mem+4" %}
  9639   opcode(0x03, 0x13);
  9640   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
  9641   ins_pipe( ialu_reg_long_mem );
  9642 %}
  9644 // Subtract Long Register with Register.
  9645 instruct subL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
  9646   match(Set dst (SubL dst src));
  9647   effect(KILL cr);
  9648   ins_cost(200);
  9649   format %{ "SUB    $dst.lo,$src.lo\n\t"
  9650             "SBB    $dst.hi,$src.hi" %}
  9651   opcode(0x2B, 0x1B);
  9652   ins_encode( RegReg_Lo(dst, src), RegReg_Hi(dst,src) );
  9653   ins_pipe( ialu_reg_reg_long );
  9654 %}
  9656 // Subtract Long Register with Immediate
  9657 instruct subL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
  9658   match(Set dst (SubL dst src));
  9659   effect(KILL cr);
  9660   format %{ "SUB    $dst.lo,$src.lo\n\t"
  9661             "SBB    $dst.hi,$src.hi" %}
  9662   opcode(0x81,0x05,0x03);  /* Opcode 81 /5, 81 /3 */
  9663   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
  9664   ins_pipe( ialu_reg_long );
  9665 %}
  9667 // Subtract Long Register with Memory
  9668 instruct subL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
  9669   match(Set dst (SubL dst (LoadL mem)));
  9670   effect(KILL cr);
  9671   ins_cost(125);
  9672   format %{ "SUB    $dst.lo,$mem\n\t"
  9673             "SBB    $dst.hi,$mem+4" %}
  9674   opcode(0x2B, 0x1B);
  9675   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
  9676   ins_pipe( ialu_reg_long_mem );
  9677 %}
  9679 instruct negL_eReg(eRegL dst, immL0 zero, eFlagsReg cr) %{
  9680   match(Set dst (SubL zero dst));
  9681   effect(KILL cr);
  9682   ins_cost(300);
  9683   format %{ "NEG    $dst.hi\n\tNEG    $dst.lo\n\tSBB    $dst.hi,0" %}
  9684   ins_encode( neg_long(dst) );
  9685   ins_pipe( ialu_reg_reg_long );
  9686 %}
  9688 // And Long Register with Register
  9689 instruct andL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
  9690   match(Set dst (AndL dst src));
  9691   effect(KILL cr);
  9692   format %{ "AND    $dst.lo,$src.lo\n\t"
  9693             "AND    $dst.hi,$src.hi" %}
  9694   opcode(0x23,0x23);
  9695   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
  9696   ins_pipe( ialu_reg_reg_long );
  9697 %}
  9699 // And Long Register with Immediate
  9700 instruct andL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
  9701   match(Set dst (AndL dst src));
  9702   effect(KILL cr);
  9703   format %{ "AND    $dst.lo,$src.lo\n\t"
  9704             "AND    $dst.hi,$src.hi" %}
  9705   opcode(0x81,0x04,0x04);  /* Opcode 81 /4, 81 /4 */
  9706   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
  9707   ins_pipe( ialu_reg_long );
  9708 %}
  9710 // And Long Register with Memory
  9711 instruct andL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
  9712   match(Set dst (AndL dst (LoadL mem)));
  9713   effect(KILL cr);
  9714   ins_cost(125);
  9715   format %{ "AND    $dst.lo,$mem\n\t"
  9716             "AND    $dst.hi,$mem+4" %}
  9717   opcode(0x23, 0x23);
  9718   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
  9719   ins_pipe( ialu_reg_long_mem );
  9720 %}
  9722 // Or Long Register with Register
  9723 instruct orl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
  9724   match(Set dst (OrL dst src));
  9725   effect(KILL cr);
  9726   format %{ "OR     $dst.lo,$src.lo\n\t"
  9727             "OR     $dst.hi,$src.hi" %}
  9728   opcode(0x0B,0x0B);
  9729   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
  9730   ins_pipe( ialu_reg_reg_long );
  9731 %}
  9733 // Or Long Register with Immediate
  9734 instruct orl_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
  9735   match(Set dst (OrL dst src));
  9736   effect(KILL cr);
  9737   format %{ "OR     $dst.lo,$src.lo\n\t"
  9738             "OR     $dst.hi,$src.hi" %}
  9739   opcode(0x81,0x01,0x01);  /* Opcode 81 /1, 81 /1 */
  9740   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
  9741   ins_pipe( ialu_reg_long );
  9742 %}
  9744 // Or Long Register with Memory
  9745 instruct orl_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
  9746   match(Set dst (OrL dst (LoadL mem)));
  9747   effect(KILL cr);
  9748   ins_cost(125);
  9749   format %{ "OR     $dst.lo,$mem\n\t"
  9750             "OR     $dst.hi,$mem+4" %}
  9751   opcode(0x0B,0x0B);
  9752   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
  9753   ins_pipe( ialu_reg_long_mem );
  9754 %}
  9756 // Xor Long Register with Register
  9757 instruct xorl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
  9758   match(Set dst (XorL dst src));
  9759   effect(KILL cr);
  9760   format %{ "XOR    $dst.lo,$src.lo\n\t"
  9761             "XOR    $dst.hi,$src.hi" %}
  9762   opcode(0x33,0x33);
  9763   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
  9764   ins_pipe( ialu_reg_reg_long );
  9765 %}
  9767 // Xor Long Register with Immediate -1
  9768 instruct xorl_eReg_im1(eRegL dst, immL_M1 imm) %{
  9769   match(Set dst (XorL dst imm));  
  9770   format %{ "NOT    $dst.lo\n\t"
  9771             "NOT    $dst.hi" %}
  9772   ins_encode %{
  9773      __ notl($dst$$Register);
  9774      __ notl(HIGH_FROM_LOW($dst$$Register));
  9775   %}
  9776   ins_pipe( ialu_reg_long );
  9777 %}
  9779 // Xor Long Register with Immediate
  9780 instruct xorl_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
  9781   match(Set dst (XorL dst src));
  9782   effect(KILL cr);
  9783   format %{ "XOR    $dst.lo,$src.lo\n\t"
  9784             "XOR    $dst.hi,$src.hi" %}
  9785   opcode(0x81,0x06,0x06);  /* Opcode 81 /6, 81 /6 */
  9786   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
  9787   ins_pipe( ialu_reg_long );
  9788 %}
  9790 // Xor Long Register with Memory
  9791 instruct xorl_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
  9792   match(Set dst (XorL dst (LoadL mem)));
  9793   effect(KILL cr);
  9794   ins_cost(125);
  9795   format %{ "XOR    $dst.lo,$mem\n\t"
  9796             "XOR    $dst.hi,$mem+4" %}
  9797   opcode(0x33,0x33);
  9798   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
  9799   ins_pipe( ialu_reg_long_mem );
  9800 %}
  9802 // Shift Left Long by 1
  9803 instruct shlL_eReg_1(eRegL dst, immI_1 cnt, eFlagsReg cr) %{
  9804   predicate(UseNewLongLShift);
  9805   match(Set dst (LShiftL dst cnt));
  9806   effect(KILL cr);
  9807   ins_cost(100);
  9808   format %{ "ADD    $dst.lo,$dst.lo\n\t"
  9809             "ADC    $dst.hi,$dst.hi" %}
  9810   ins_encode %{
  9811     __ addl($dst$$Register,$dst$$Register);
  9812     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
  9813   %}
  9814   ins_pipe( ialu_reg_long );
  9815 %}
  9817 // Shift Left Long by 2
  9818 instruct shlL_eReg_2(eRegL dst, immI_2 cnt, eFlagsReg cr) %{
  9819   predicate(UseNewLongLShift);
  9820   match(Set dst (LShiftL dst cnt));
  9821   effect(KILL cr);
  9822   ins_cost(100);
  9823   format %{ "ADD    $dst.lo,$dst.lo\n\t"
  9824             "ADC    $dst.hi,$dst.hi\n\t" 
  9825             "ADD    $dst.lo,$dst.lo\n\t"
  9826             "ADC    $dst.hi,$dst.hi" %}
  9827   ins_encode %{
  9828     __ addl($dst$$Register,$dst$$Register);
  9829     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
  9830     __ addl($dst$$Register,$dst$$Register);
  9831     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
  9832   %}
  9833   ins_pipe( ialu_reg_long );
  9834 %}
  9836 // Shift Left Long by 3
  9837 instruct shlL_eReg_3(eRegL dst, immI_3 cnt, eFlagsReg cr) %{
  9838   predicate(UseNewLongLShift);
  9839   match(Set dst (LShiftL dst cnt));
  9840   effect(KILL cr);
  9841   ins_cost(100);
  9842   format %{ "ADD    $dst.lo,$dst.lo\n\t"
  9843             "ADC    $dst.hi,$dst.hi\n\t" 
  9844             "ADD    $dst.lo,$dst.lo\n\t"
  9845             "ADC    $dst.hi,$dst.hi\n\t" 
  9846             "ADD    $dst.lo,$dst.lo\n\t"
  9847             "ADC    $dst.hi,$dst.hi" %}
  9848   ins_encode %{
  9849     __ addl($dst$$Register,$dst$$Register);
  9850     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
  9851     __ addl($dst$$Register,$dst$$Register);
  9852     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
  9853     __ addl($dst$$Register,$dst$$Register);
  9854     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
  9855   %}
  9856   ins_pipe( ialu_reg_long );
  9857 %}
  9859 // Shift Left Long by 1-31
  9860 instruct shlL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
  9861   match(Set dst (LShiftL dst cnt));
  9862   effect(KILL cr);
  9863   ins_cost(200);
  9864   format %{ "SHLD   $dst.hi,$dst.lo,$cnt\n\t"
  9865             "SHL    $dst.lo,$cnt" %}
  9866   opcode(0xC1, 0x4, 0xA4);  /* 0F/A4, then C1 /4 ib */
  9867   ins_encode( move_long_small_shift(dst,cnt) );
  9868   ins_pipe( ialu_reg_long );
  9869 %}
  9871 // Shift Left Long by 32-63
  9872 instruct shlL_eReg_32_63(eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
  9873   match(Set dst (LShiftL dst cnt));
  9874   effect(KILL cr);
  9875   ins_cost(300);
  9876   format %{ "MOV    $dst.hi,$dst.lo\n"
  9877           "\tSHL    $dst.hi,$cnt-32\n"
  9878           "\tXOR    $dst.lo,$dst.lo" %}
  9879   opcode(0xC1, 0x4);  /* C1 /4 ib */
  9880   ins_encode( move_long_big_shift_clr(dst,cnt) );
  9881   ins_pipe( ialu_reg_long );
  9882 %}
  9884 // Shift Left Long by variable
  9885 instruct salL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
  9886   match(Set dst (LShiftL dst shift));
  9887   effect(KILL cr);
  9888   ins_cost(500+200);
  9889   size(17);
  9890   format %{ "TEST   $shift,32\n\t"
  9891             "JEQ,s  small\n\t"
  9892             "MOV    $dst.hi,$dst.lo\n\t"
  9893             "XOR    $dst.lo,$dst.lo\n"
  9894     "small:\tSHLD   $dst.hi,$dst.lo,$shift\n\t"
  9895             "SHL    $dst.lo,$shift" %}
  9896   ins_encode( shift_left_long( dst, shift ) );
  9897   ins_pipe( pipe_slow );
  9898 %}
  9900 // Shift Right Long by 1-31
  9901 instruct shrL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
  9902   match(Set dst (URShiftL dst cnt));
  9903   effect(KILL cr);
  9904   ins_cost(200);
  9905   format %{ "SHRD   $dst.lo,$dst.hi,$cnt\n\t"
  9906             "SHR    $dst.hi,$cnt" %}
  9907   opcode(0xC1, 0x5, 0xAC);  /* 0F/AC, then C1 /5 ib */
  9908   ins_encode( move_long_small_shift(dst,cnt) );
  9909   ins_pipe( ialu_reg_long );
  9910 %}
  9912 // Shift Right Long by 32-63
  9913 instruct shrL_eReg_32_63(eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
  9914   match(Set dst (URShiftL dst cnt));
  9915   effect(KILL cr);
  9916   ins_cost(300);
  9917   format %{ "MOV    $dst.lo,$dst.hi\n"
  9918           "\tSHR    $dst.lo,$cnt-32\n"
  9919           "\tXOR    $dst.hi,$dst.hi" %}
  9920   opcode(0xC1, 0x5);  /* C1 /5 ib */
  9921   ins_encode( move_long_big_shift_clr(dst,cnt) );
  9922   ins_pipe( ialu_reg_long );
  9923 %}
  9925 // Shift Right Long by variable
  9926 instruct shrL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
  9927   match(Set dst (URShiftL dst shift));
  9928   effect(KILL cr);
  9929   ins_cost(600);
  9930   size(17);
  9931   format %{ "TEST   $shift,32\n\t"
  9932             "JEQ,s  small\n\t"
  9933             "MOV    $dst.lo,$dst.hi\n\t"
  9934             "XOR    $dst.hi,$dst.hi\n"
  9935     "small:\tSHRD   $dst.lo,$dst.hi,$shift\n\t"
  9936             "SHR    $dst.hi,$shift" %}
  9937   ins_encode( shift_right_long( dst, shift ) );
  9938   ins_pipe( pipe_slow );
  9939 %}
  9941 // Shift Right Long by 1-31
  9942 instruct sarL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
  9943   match(Set dst (RShiftL dst cnt));
  9944   effect(KILL cr);
  9945   ins_cost(200);
  9946   format %{ "SHRD   $dst.lo,$dst.hi,$cnt\n\t"
  9947             "SAR    $dst.hi,$cnt" %}
  9948   opcode(0xC1, 0x7, 0xAC);  /* 0F/AC, then C1 /7 ib */
  9949   ins_encode( move_long_small_shift(dst,cnt) );
  9950   ins_pipe( ialu_reg_long );
  9951 %}
  9953 // Shift Right Long by 32-63
  9954 instruct sarL_eReg_32_63( eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
  9955   match(Set dst (RShiftL dst cnt));
  9956   effect(KILL cr);
  9957   ins_cost(300);
  9958   format %{ "MOV    $dst.lo,$dst.hi\n"
  9959           "\tSAR    $dst.lo,$cnt-32\n"
  9960           "\tSAR    $dst.hi,31" %}
  9961   opcode(0xC1, 0x7);  /* C1 /7 ib */
  9962   ins_encode( move_long_big_shift_sign(dst,cnt) );
  9963   ins_pipe( ialu_reg_long );
  9964 %}
  9966 // Shift Right arithmetic Long by variable
  9967 instruct sarL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
  9968   match(Set dst (RShiftL dst shift));
  9969   effect(KILL cr);
  9970   ins_cost(600);
  9971   size(18);
  9972   format %{ "TEST   $shift,32\n\t"
  9973             "JEQ,s  small\n\t"
  9974             "MOV    $dst.lo,$dst.hi\n\t"
  9975             "SAR    $dst.hi,31\n"
  9976     "small:\tSHRD   $dst.lo,$dst.hi,$shift\n\t"
  9977             "SAR    $dst.hi,$shift" %}
  9978   ins_encode( shift_right_arith_long( dst, shift ) );
  9979   ins_pipe( pipe_slow );
  9980 %}
  9983 //----------Double Instructions------------------------------------------------
  9984 // Double Math
  9986 // Compare & branch
  9988 // P6 version of float compare, sets condition codes in EFLAGS
  9989 instruct cmpD_cc_P6(eFlagsRegU cr, regD src1, regD src2, eAXRegI rax) %{
  9990   predicate(VM_Version::supports_cmov() && UseSSE <=1);
  9991   match(Set cr (CmpD src1 src2));
  9992   effect(KILL rax);
  9993   ins_cost(150);
  9994   format %{ "FLD    $src1\n\t"
  9995             "FUCOMIP ST,$src2  // P6 instruction\n\t"
  9996             "JNP    exit\n\t"
  9997             "MOV    ah,1       // saw a NaN, set CF\n\t"
  9998             "SAHF\n"
  9999      "exit:\tNOP               // avoid branch to branch" %}
 10000   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
 10001   ins_encode( Push_Reg_D(src1),
 10002               OpcP, RegOpc(src2),
 10003               cmpF_P6_fixup );
 10004   ins_pipe( pipe_slow );
 10005 %}
 10007 instruct cmpD_cc_P6CF(eFlagsRegUCF cr, regD src1, regD src2) %{
 10008   predicate(VM_Version::supports_cmov() && UseSSE <=1);
 10009   match(Set cr (CmpD src1 src2));
 10010   ins_cost(150);
 10011   format %{ "FLD    $src1\n\t"
 10012             "FUCOMIP ST,$src2  // P6 instruction" %}
 10013   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
 10014   ins_encode( Push_Reg_D(src1),
 10015               OpcP, RegOpc(src2));
 10016   ins_pipe( pipe_slow );
 10017 %}
 10019 // Compare & branch
 10020 instruct cmpD_cc(eFlagsRegU cr, regD src1, regD src2, eAXRegI rax) %{
 10021   predicate(UseSSE<=1);
 10022   match(Set cr (CmpD src1 src2));
 10023   effect(KILL rax);
 10024   ins_cost(200);
 10025   format %{ "FLD    $src1\n\t"
 10026             "FCOMp  $src2\n\t"
 10027             "FNSTSW AX\n\t"
 10028             "TEST   AX,0x400\n\t"
 10029             "JZ,s   flags\n\t"
 10030             "MOV    AH,1\t# unordered treat as LT\n"
 10031     "flags:\tSAHF" %}
 10032   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
 10033   ins_encode( Push_Reg_D(src1),
 10034               OpcP, RegOpc(src2),
 10035               fpu_flags);
 10036   ins_pipe( pipe_slow );
 10037 %}
 10039 // Compare vs zero into -1,0,1
 10040 instruct cmpD_0(eRegI dst, regD src1, immD0 zero, eAXRegI rax, eFlagsReg cr) %{
 10041   predicate(UseSSE<=1);
 10042   match(Set dst (CmpD3 src1 zero));
 10043   effect(KILL cr, KILL rax);
 10044   ins_cost(280);
 10045   format %{ "FTSTD  $dst,$src1" %}
 10046   opcode(0xE4, 0xD9);
 10047   ins_encode( Push_Reg_D(src1),
 10048               OpcS, OpcP, PopFPU,
 10049               CmpF_Result(dst));
 10050   ins_pipe( pipe_slow );
 10051 %}
 10053 // Compare into -1,0,1
 10054 instruct cmpD_reg(eRegI dst, regD src1, regD src2, eAXRegI rax, eFlagsReg cr) %{
 10055   predicate(UseSSE<=1);
 10056   match(Set dst (CmpD3 src1 src2));
 10057   effect(KILL cr, KILL rax);
 10058   ins_cost(300);
 10059   format %{ "FCMPD  $dst,$src1,$src2" %}
 10060   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
 10061   ins_encode( Push_Reg_D(src1),
 10062               OpcP, RegOpc(src2),
 10063               CmpF_Result(dst));
 10064   ins_pipe( pipe_slow );
 10065 %}
 10067 // float compare and set condition codes in EFLAGS by XMM regs
 10068 instruct cmpXD_cc(eFlagsRegU cr, regXD dst, regXD src, eAXRegI rax) %{
 10069   predicate(UseSSE>=2);
 10070   match(Set cr (CmpD dst src));
 10071   effect(KILL rax);
 10072   ins_cost(125);
 10073   format %{ "COMISD $dst,$src\n"
 10074           "\tJNP    exit\n"
 10075           "\tMOV    ah,1       // saw a NaN, set CF\n"
 10076           "\tSAHF\n"
 10077      "exit:\tNOP               // avoid branch to branch" %}
 10078   opcode(0x66, 0x0F, 0x2F);
 10079   ins_encode(OpcP, OpcS, Opcode(tertiary), RegReg(dst, src), cmpF_P6_fixup);
 10080   ins_pipe( pipe_slow );
 10081 %}
 10083 instruct cmpXD_ccCF(eFlagsRegUCF cr, regXD dst, regXD src) %{
 10084   predicate(UseSSE>=2);
 10085   match(Set cr (CmpD dst src));
 10086   ins_cost(100);
 10087   format %{ "COMISD $dst,$src" %}
 10088   opcode(0x66, 0x0F, 0x2F);
 10089   ins_encode(OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
 10090   ins_pipe( pipe_slow );
 10091 %}
 10093 // float compare and set condition codes in EFLAGS by XMM regs
 10094 instruct cmpXD_ccmem(eFlagsRegU cr, regXD dst, memory src, eAXRegI rax) %{
 10095   predicate(UseSSE>=2);
 10096   match(Set cr (CmpD dst (LoadD src)));
 10097   effect(KILL rax);
 10098   ins_cost(145);
 10099   format %{ "COMISD $dst,$src\n"
 10100           "\tJNP    exit\n"
 10101           "\tMOV    ah,1       // saw a NaN, set CF\n"
 10102           "\tSAHF\n"
 10103      "exit:\tNOP               // avoid branch to branch" %}
 10104   opcode(0x66, 0x0F, 0x2F);
 10105   ins_encode(OpcP, OpcS, Opcode(tertiary), RegMem(dst, src), cmpF_P6_fixup);
 10106   ins_pipe( pipe_slow );
 10107 %}
 10109 instruct cmpXD_ccmemCF(eFlagsRegUCF cr, regXD dst, memory src) %{
 10110   predicate(UseSSE>=2);
 10111   match(Set cr (CmpD dst (LoadD src)));
 10112   ins_cost(100);
 10113   format %{ "COMISD $dst,$src" %}
 10114   opcode(0x66, 0x0F, 0x2F);
 10115   ins_encode(OpcP, OpcS, Opcode(tertiary), RegMem(dst, src));
 10116   ins_pipe( pipe_slow );
 10117 %}
 10119 // Compare into -1,0,1 in XMM
 10120 instruct cmpXD_reg(eRegI dst, regXD src1, regXD src2, eFlagsReg cr) %{
 10121   predicate(UseSSE>=2);
 10122   match(Set dst (CmpD3 src1 src2));
 10123   effect(KILL cr);
 10124   ins_cost(255);
 10125   format %{ "XOR    $dst,$dst\n"
 10126           "\tCOMISD $src1,$src2\n"
 10127           "\tJP,s   nan\n"
 10128           "\tJEQ,s  exit\n"
 10129           "\tJA,s   inc\n"
 10130       "nan:\tDEC    $dst\n"
 10131           "\tJMP,s  exit\n"
 10132       "inc:\tINC    $dst\n"
 10133       "exit:"
 10134                 %}
 10135   opcode(0x66, 0x0F, 0x2F);
 10136   ins_encode(Xor_Reg(dst), OpcP, OpcS, Opcode(tertiary), RegReg(src1, src2),
 10137              CmpX_Result(dst));
 10138   ins_pipe( pipe_slow );
 10139 %}
 10141 // Compare into -1,0,1 in XMM and memory
 10142 instruct cmpXD_regmem(eRegI dst, regXD src1, memory mem, eFlagsReg cr) %{
 10143   predicate(UseSSE>=2);
 10144   match(Set dst (CmpD3 src1 (LoadD mem)));
 10145   effect(KILL cr);
 10146   ins_cost(275);
 10147   format %{ "COMISD $src1,$mem\n"
 10148           "\tMOV    $dst,0\t\t# do not blow flags\n"
 10149           "\tJP,s   nan\n"
 10150           "\tJEQ,s  exit\n"
 10151           "\tJA,s   inc\n"
 10152       "nan:\tDEC    $dst\n"
 10153           "\tJMP,s  exit\n"
 10154       "inc:\tINC    $dst\n"
 10155       "exit:"
 10156                 %}
 10157   opcode(0x66, 0x0F, 0x2F);
 10158   ins_encode(OpcP, OpcS, Opcode(tertiary), RegMem(src1, mem),
 10159              LdImmI(dst,0x0), CmpX_Result(dst));
 10160   ins_pipe( pipe_slow );
 10161 %}
 10164 instruct subD_reg(regD dst, regD src) %{
 10165   predicate (UseSSE <=1);
 10166   match(Set dst (SubD dst src));
 10168   format %{ "FLD    $src\n\t"
 10169             "DSUBp  $dst,ST" %}
 10170   opcode(0xDE, 0x5); /* DE E8+i  or DE /5 */
 10171   ins_cost(150);
 10172   ins_encode( Push_Reg_D(src),
 10173               OpcP, RegOpc(dst) );
 10174   ins_pipe( fpu_reg_reg );
 10175 %}
 10177 instruct subD_reg_round(stackSlotD dst, regD src1, regD src2) %{
 10178   predicate (UseSSE <=1);
 10179   match(Set dst (RoundDouble (SubD src1 src2)));
 10180   ins_cost(250);
 10182   format %{ "FLD    $src2\n\t"
 10183             "DSUB   ST,$src1\n\t"
 10184             "FSTP_D $dst\t# D-round" %}
 10185   opcode(0xD8, 0x5);
 10186   ins_encode( Push_Reg_D(src2),
 10187               OpcP, RegOpc(src1), Pop_Mem_D(dst) );
 10188   ins_pipe( fpu_mem_reg_reg );
 10189 %}
 10192 instruct subD_reg_mem(regD dst, memory src) %{
 10193   predicate (UseSSE <=1);
 10194   match(Set dst (SubD dst (LoadD src)));
 10195   ins_cost(150);
 10197   format %{ "FLD    $src\n\t"
 10198             "DSUBp  $dst,ST" %}
 10199   opcode(0xDE, 0x5, 0xDD); /* DE C0+i */  /* LoadD  DD /0 */
 10200   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
 10201               OpcP, RegOpc(dst) );
 10202   ins_pipe( fpu_reg_mem );
 10203 %}
 10205 instruct absD_reg(regDPR1 dst, regDPR1 src) %{
 10206   predicate (UseSSE<=1);
 10207   match(Set dst (AbsD src));
 10208   ins_cost(100);
 10209   format %{ "FABS" %}
 10210   opcode(0xE1, 0xD9);
 10211   ins_encode( OpcS, OpcP );
 10212   ins_pipe( fpu_reg_reg );
 10213 %}
 10215 instruct absXD_reg( regXD dst ) %{
 10216   predicate(UseSSE>=2);
 10217   match(Set dst (AbsD dst));
 10218   format %{ "ANDPD  $dst,[0x7FFFFFFFFFFFFFFF]\t# ABS D by sign masking" %}
 10219   ins_encode( AbsXD_encoding(dst));
 10220   ins_pipe( pipe_slow );
 10221 %}
 10223 instruct negD_reg(regDPR1 dst, regDPR1 src) %{
 10224   predicate(UseSSE<=1);
 10225   match(Set dst (NegD src));
 10226   ins_cost(100);
 10227   format %{ "FCHS" %}
 10228   opcode(0xE0, 0xD9);
 10229   ins_encode( OpcS, OpcP );
 10230   ins_pipe( fpu_reg_reg );
 10231 %}
 10233 instruct negXD_reg( regXD dst ) %{
 10234   predicate(UseSSE>=2);
 10235   match(Set dst (NegD dst));
 10236   format %{ "XORPD  $dst,[0x8000000000000000]\t# CHS D by sign flipping" %}
 10237   ins_encode %{
 10238      __ xorpd($dst$$XMMRegister,
 10239               ExternalAddress((address)double_signflip_pool));
 10240   %}
 10241   ins_pipe( pipe_slow );
 10242 %}
 10244 instruct addD_reg(regD dst, regD src) %{
 10245   predicate(UseSSE<=1);
 10246   match(Set dst (AddD dst src));
 10247   format %{ "FLD    $src\n\t"
 10248             "DADD   $dst,ST" %}
 10249   size(4);
 10250   ins_cost(150);
 10251   opcode(0xDE, 0x0); /* DE C0+i or DE /0*/
 10252   ins_encode( Push_Reg_D(src),
 10253               OpcP, RegOpc(dst) );
 10254   ins_pipe( fpu_reg_reg );
 10255 %}
 10258 instruct addD_reg_round(stackSlotD dst, regD src1, regD src2) %{
 10259   predicate(UseSSE<=1);
 10260   match(Set dst (RoundDouble (AddD src1 src2)));
 10261   ins_cost(250);
 10263   format %{ "FLD    $src2\n\t"
 10264             "DADD   ST,$src1\n\t"
 10265             "FSTP_D $dst\t# D-round" %}
 10266   opcode(0xD8, 0x0); /* D8 C0+i or D8 /0*/
 10267   ins_encode( Push_Reg_D(src2),
 10268               OpcP, RegOpc(src1), Pop_Mem_D(dst) );
 10269   ins_pipe( fpu_mem_reg_reg );
 10270 %}
 10273 instruct addD_reg_mem(regD dst, memory src) %{
 10274   predicate(UseSSE<=1);
 10275   match(Set dst (AddD dst (LoadD src)));
 10276   ins_cost(150);
 10278   format %{ "FLD    $src\n\t"
 10279             "DADDp  $dst,ST" %}
 10280   opcode(0xDE, 0x0, 0xDD); /* DE C0+i */  /* LoadD  DD /0 */
 10281   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
 10282               OpcP, RegOpc(dst) );
 10283   ins_pipe( fpu_reg_mem );
 10284 %}
 10286 // add-to-memory
 10287 instruct addD_mem_reg(memory dst, regD src) %{
 10288   predicate(UseSSE<=1);
 10289   match(Set dst (StoreD dst (RoundDouble (AddD (LoadD dst) src))));
 10290   ins_cost(150);
 10292   format %{ "FLD_D  $dst\n\t"
 10293             "DADD   ST,$src\n\t"
 10294             "FST_D  $dst" %}
 10295   opcode(0xDD, 0x0);
 10296   ins_encode( Opcode(0xDD), RMopc_Mem(0x00,dst),
 10297               Opcode(0xD8), RegOpc(src),
 10298               set_instruction_start,
 10299               Opcode(0xDD), RMopc_Mem(0x03,dst) );
 10300   ins_pipe( fpu_reg_mem );
 10301 %}
 10303 instruct addD_reg_imm1(regD dst, immD1 con) %{
 10304   predicate(UseSSE<=1);
 10305   match(Set dst (AddD dst con));
 10306   ins_cost(125);
 10307   format %{ "FLD1\n\t"
 10308             "DADDp  $dst,ST" %}
 10309   ins_encode %{
 10310     __ fld1();
 10311     __ faddp($dst$$reg);
 10312   %}
 10313   ins_pipe(fpu_reg);
 10314 %}
 10316 instruct addD_reg_imm(regD dst, immD con) %{
 10317   predicate(UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
 10318   match(Set dst (AddD dst con));
 10319   ins_cost(200);
 10320   format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
 10321             "DADDp  $dst,ST" %}
 10322   ins_encode %{
 10323     __ fld_d($constantaddress($con));
 10324     __ faddp($dst$$reg);
 10325   %}
 10326   ins_pipe(fpu_reg_mem);
 10327 %}
 10329 instruct addD_reg_imm_round(stackSlotD dst, regD src, immD con) %{
 10330   predicate(UseSSE<=1 && _kids[0]->_kids[1]->_leaf->getd() != 0.0 && _kids[0]->_kids[1]->_leaf->getd() != 1.0 );
 10331   match(Set dst (RoundDouble (AddD src con)));
 10332   ins_cost(200);
 10333   format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
 10334             "DADD   ST,$src\n\t"
 10335             "FSTP_D $dst\t# D-round" %}
 10336   ins_encode %{
 10337     __ fld_d($constantaddress($con));
 10338     __ fadd($src$$reg);
 10339     __ fstp_d(Address(rsp, $dst$$disp));
 10340   %}
 10341   ins_pipe(fpu_mem_reg_con);
 10342 %}
 10344 // Add two double precision floating point values in xmm
 10345 instruct addXD_reg(regXD dst, regXD src) %{
 10346   predicate(UseSSE>=2);
 10347   match(Set dst (AddD dst src));
 10348   format %{ "ADDSD  $dst,$src" %}
 10349   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x58), RegReg(dst, src));
 10350   ins_pipe( pipe_slow );
 10351 %}
 10353 instruct addXD_imm(regXD dst, immXD con) %{
 10354   predicate(UseSSE>=2);
 10355   match(Set dst (AddD dst con));
 10356   format %{ "ADDSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
 10357   ins_encode %{
 10358     __ addsd($dst$$XMMRegister, $constantaddress($con));
 10359   %}
 10360   ins_pipe(pipe_slow);
 10361 %}
 10363 instruct addXD_mem(regXD dst, memory mem) %{
 10364   predicate(UseSSE>=2);
 10365   match(Set dst (AddD dst (LoadD mem)));
 10366   format %{ "ADDSD  $dst,$mem" %}
 10367   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x58), RegMem(dst,mem));
 10368   ins_pipe( pipe_slow );
 10369 %}
 10371 // Sub two double precision floating point values in xmm
 10372 instruct subXD_reg(regXD dst, regXD src) %{
 10373   predicate(UseSSE>=2);
 10374   match(Set dst (SubD dst src));
 10375   format %{ "SUBSD  $dst,$src" %}
 10376   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5C), RegReg(dst, src));
 10377   ins_pipe( pipe_slow );
 10378 %}
 10380 instruct subXD_imm(regXD dst, immXD con) %{
 10381   predicate(UseSSE>=2);
 10382   match(Set dst (SubD dst con));
 10383   format %{ "SUBSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
 10384   ins_encode %{
 10385     __ subsd($dst$$XMMRegister, $constantaddress($con));
 10386   %}
 10387   ins_pipe(pipe_slow);
 10388 %}
 10390 instruct subXD_mem(regXD dst, memory mem) %{
 10391   predicate(UseSSE>=2);
 10392   match(Set dst (SubD dst (LoadD mem)));
 10393   format %{ "SUBSD  $dst,$mem" %}
 10394   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5C), RegMem(dst,mem));
 10395   ins_pipe( pipe_slow );
 10396 %}
 10398 // Mul two double precision floating point values in xmm
 10399 instruct mulXD_reg(regXD dst, regXD src) %{
 10400   predicate(UseSSE>=2);
 10401   match(Set dst (MulD dst src));
 10402   format %{ "MULSD  $dst,$src" %}
 10403   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x59), RegReg(dst, src));
 10404   ins_pipe( pipe_slow );
 10405 %}
 10407 instruct mulXD_imm(regXD dst, immXD con) %{
 10408   predicate(UseSSE>=2);
 10409   match(Set dst (MulD dst con));
 10410   format %{ "MULSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
 10411   ins_encode %{
 10412     __ mulsd($dst$$XMMRegister, $constantaddress($con));
 10413   %}
 10414   ins_pipe(pipe_slow);
 10415 %}
 10417 instruct mulXD_mem(regXD dst, memory mem) %{
 10418   predicate(UseSSE>=2);
 10419   match(Set dst (MulD dst (LoadD mem)));
 10420   format %{ "MULSD  $dst,$mem" %}
 10421   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x59), RegMem(dst,mem));
 10422   ins_pipe( pipe_slow );
 10423 %}
 10425 // Div two double precision floating point values in xmm
 10426 instruct divXD_reg(regXD dst, regXD src) %{
 10427   predicate(UseSSE>=2);
 10428   match(Set dst (DivD dst src));
 10429   format %{ "DIVSD  $dst,$src" %}
 10430   opcode(0xF2, 0x0F, 0x5E);
 10431   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5E), RegReg(dst, src));
 10432   ins_pipe( pipe_slow );
 10433 %}
 10435 instruct divXD_imm(regXD dst, immXD con) %{
 10436   predicate(UseSSE>=2);
 10437   match(Set dst (DivD dst con));
 10438   format %{ "DIVSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
 10439   ins_encode %{
 10440     __ divsd($dst$$XMMRegister, $constantaddress($con));
 10441   %}
 10442   ins_pipe(pipe_slow);
 10443 %}
 10445 instruct divXD_mem(regXD dst, memory mem) %{
 10446   predicate(UseSSE>=2);
 10447   match(Set dst (DivD dst (LoadD mem)));
 10448   format %{ "DIVSD  $dst,$mem" %}
 10449   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5E), RegMem(dst,mem));
 10450   ins_pipe( pipe_slow );
 10451 %}
 10454 instruct mulD_reg(regD dst, regD src) %{
 10455   predicate(UseSSE<=1);
 10456   match(Set dst (MulD dst src));
 10457   format %{ "FLD    $src\n\t"
 10458             "DMULp  $dst,ST" %}
 10459   opcode(0xDE, 0x1); /* DE C8+i or DE /1*/
 10460   ins_cost(150);
 10461   ins_encode( Push_Reg_D(src),
 10462               OpcP, RegOpc(dst) );
 10463   ins_pipe( fpu_reg_reg );
 10464 %}
 10466 // Strict FP instruction biases argument before multiply then
 10467 // biases result to avoid double rounding of subnormals.
 10468 //
 10469 // scale arg1 by multiplying arg1 by 2^(-15360)
 10470 // load arg2
 10471 // multiply scaled arg1 by arg2
 10472 // rescale product by 2^(15360)
 10473 //
 10474 instruct strictfp_mulD_reg(regDPR1 dst, regnotDPR1 src) %{
 10475   predicate( UseSSE<=1 && Compile::current()->has_method() && Compile::current()->method()->is_strict() );
 10476   match(Set dst (MulD dst src));
 10477   ins_cost(1);   // Select this instruction for all strict FP double multiplies
 10479   format %{ "FLD    StubRoutines::_fpu_subnormal_bias1\n\t"
 10480             "DMULp  $dst,ST\n\t"
 10481             "FLD    $src\n\t"
 10482             "DMULp  $dst,ST\n\t"
 10483             "FLD    StubRoutines::_fpu_subnormal_bias2\n\t"
 10484             "DMULp  $dst,ST\n\t" %}
 10485   opcode(0xDE, 0x1); /* DE C8+i or DE /1*/
 10486   ins_encode( strictfp_bias1(dst),
 10487               Push_Reg_D(src),
 10488               OpcP, RegOpc(dst),
 10489               strictfp_bias2(dst) );
 10490   ins_pipe( fpu_reg_reg );
 10491 %}
 10493 instruct mulD_reg_imm(regD dst, immD con) %{
 10494   predicate( UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
 10495   match(Set dst (MulD dst con));
 10496   ins_cost(200);
 10497   format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
 10498             "DMULp  $dst,ST" %}
 10499   ins_encode %{
 10500     __ fld_d($constantaddress($con));
 10501     __ fmulp($dst$$reg);
 10502   %}
 10503   ins_pipe(fpu_reg_mem);
 10504 %}
 10507 instruct mulD_reg_mem(regD dst, memory src) %{
 10508   predicate( UseSSE<=1 );
 10509   match(Set dst (MulD dst (LoadD src)));
 10510   ins_cost(200);
 10511   format %{ "FLD_D  $src\n\t"
 10512             "DMULp  $dst,ST" %}
 10513   opcode(0xDE, 0x1, 0xDD); /* DE C8+i or DE /1*/  /* LoadD  DD /0 */
 10514   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
 10515               OpcP, RegOpc(dst) );
 10516   ins_pipe( fpu_reg_mem );
 10517 %}
 10519 //
 10520 // Cisc-alternate to reg-reg multiply
 10521 instruct mulD_reg_mem_cisc(regD dst, regD src, memory mem) %{
 10522   predicate( UseSSE<=1 );
 10523   match(Set dst (MulD src (LoadD mem)));
 10524   ins_cost(250);
 10525   format %{ "FLD_D  $mem\n\t"
 10526             "DMUL   ST,$src\n\t"
 10527             "FSTP_D $dst" %}
 10528   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i */  /* LoadD D9 /0 */
 10529   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,mem),
 10530               OpcReg_F(src),
 10531               Pop_Reg_D(dst) );
 10532   ins_pipe( fpu_reg_reg_mem );
 10533 %}
 10536 // MACRO3 -- addD a mulD
 10537 // This instruction is a '2-address' instruction in that the result goes
 10538 // back to src2.  This eliminates a move from the macro; possibly the
 10539 // register allocator will have to add it back (and maybe not).
 10540 instruct addD_mulD_reg(regD src2, regD src1, regD src0) %{
 10541   predicate( UseSSE<=1 );
 10542   match(Set src2 (AddD (MulD src0 src1) src2));
 10543   format %{ "FLD    $src0\t# ===MACRO3d===\n\t"
 10544             "DMUL   ST,$src1\n\t"
 10545             "DADDp  $src2,ST" %}
 10546   ins_cost(250);
 10547   opcode(0xDD); /* LoadD DD /0 */
 10548   ins_encode( Push_Reg_F(src0),
 10549               FMul_ST_reg(src1),
 10550               FAddP_reg_ST(src2) );
 10551   ins_pipe( fpu_reg_reg_reg );
 10552 %}
 10555 // MACRO3 -- subD a mulD
 10556 instruct subD_mulD_reg(regD src2, regD src1, regD src0) %{
 10557   predicate( UseSSE<=1 );
 10558   match(Set src2 (SubD (MulD src0 src1) src2));
 10559   format %{ "FLD    $src0\t# ===MACRO3d===\n\t"
 10560             "DMUL   ST,$src1\n\t"
 10561             "DSUBRp $src2,ST" %}
 10562   ins_cost(250);
 10563   ins_encode( Push_Reg_F(src0),
 10564               FMul_ST_reg(src1),
 10565               Opcode(0xDE), Opc_plus(0xE0,src2));
 10566   ins_pipe( fpu_reg_reg_reg );
 10567 %}
 10570 instruct divD_reg(regD dst, regD src) %{
 10571   predicate( UseSSE<=1 );
 10572   match(Set dst (DivD dst src));
 10574   format %{ "FLD    $src\n\t"
 10575             "FDIVp  $dst,ST" %}
 10576   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
 10577   ins_cost(150);
 10578   ins_encode( Push_Reg_D(src),
 10579               OpcP, RegOpc(dst) );
 10580   ins_pipe( fpu_reg_reg );
 10581 %}
 10583 // Strict FP instruction biases argument before division then
 10584 // biases result, to avoid double rounding of subnormals.
 10585 //
 10586 // scale dividend by multiplying dividend by 2^(-15360)
 10587 // load divisor
 10588 // divide scaled dividend by divisor
 10589 // rescale quotient by 2^(15360)
 10590 //
 10591 instruct strictfp_divD_reg(regDPR1 dst, regnotDPR1 src) %{
 10592   predicate (UseSSE<=1);
 10593   match(Set dst (DivD dst src));
 10594   predicate( UseSSE<=1 && Compile::current()->has_method() && Compile::current()->method()->is_strict() );
 10595   ins_cost(01);
 10597   format %{ "FLD    StubRoutines::_fpu_subnormal_bias1\n\t"
 10598             "DMULp  $dst,ST\n\t"
 10599             "FLD    $src\n\t"
 10600             "FDIVp  $dst,ST\n\t"
 10601             "FLD    StubRoutines::_fpu_subnormal_bias2\n\t"
 10602             "DMULp  $dst,ST\n\t" %}
 10603   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
 10604   ins_encode( strictfp_bias1(dst),
 10605               Push_Reg_D(src),
 10606               OpcP, RegOpc(dst),
 10607               strictfp_bias2(dst) );
 10608   ins_pipe( fpu_reg_reg );
 10609 %}
 10611 instruct divD_reg_round(stackSlotD dst, regD src1, regD src2) %{
 10612   predicate( UseSSE<=1 && !(Compile::current()->has_method() && Compile::current()->method()->is_strict()) );
 10613   match(Set dst (RoundDouble (DivD src1 src2)));
 10615   format %{ "FLD    $src1\n\t"
 10616             "FDIV   ST,$src2\n\t"
 10617             "FSTP_D $dst\t# D-round" %}
 10618   opcode(0xD8, 0x6); /* D8 F0+i or D8 /6 */
 10619   ins_encode( Push_Reg_D(src1),
 10620               OpcP, RegOpc(src2), Pop_Mem_D(dst) );
 10621   ins_pipe( fpu_mem_reg_reg );
 10622 %}
 10625 instruct modD_reg(regD dst, regD src, eAXRegI rax, eFlagsReg cr) %{
 10626   predicate(UseSSE<=1);
 10627   match(Set dst (ModD dst src));
 10628   effect(KILL rax, KILL cr); // emitModD() uses EAX and EFLAGS
 10630   format %{ "DMOD   $dst,$src" %}
 10631   ins_cost(250);
 10632   ins_encode(Push_Reg_Mod_D(dst, src),
 10633               emitModD(),
 10634               Push_Result_Mod_D(src),
 10635               Pop_Reg_D(dst));
 10636   ins_pipe( pipe_slow );
 10637 %}
 10639 instruct modXD_reg(regXD dst, regXD src0, regXD src1, eAXRegI rax, eFlagsReg cr) %{
 10640   predicate(UseSSE>=2);
 10641   match(Set dst (ModD src0 src1));
 10642   effect(KILL rax, KILL cr);
 10644   format %{ "SUB    ESP,8\t # DMOD\n"
 10645           "\tMOVSD  [ESP+0],$src1\n"
 10646           "\tFLD_D  [ESP+0]\n"
 10647           "\tMOVSD  [ESP+0],$src0\n"
 10648           "\tFLD_D  [ESP+0]\n"
 10649      "loop:\tFPREM\n"
 10650           "\tFWAIT\n"
 10651           "\tFNSTSW AX\n"
 10652           "\tSAHF\n"
 10653           "\tJP     loop\n"
 10654           "\tFSTP_D [ESP+0]\n"
 10655           "\tMOVSD  $dst,[ESP+0]\n"
 10656           "\tADD    ESP,8\n"
 10657           "\tFSTP   ST0\t # Restore FPU Stack"
 10658     %}
 10659   ins_cost(250);
 10660   ins_encode( Push_ModD_encoding(src0, src1), emitModD(), Push_ResultXD(dst), PopFPU);
 10661   ins_pipe( pipe_slow );
 10662 %}
 10664 instruct sinD_reg(regDPR1 dst, regDPR1 src) %{
 10665   predicate (UseSSE<=1);
 10666   match(Set dst (SinD src));
 10667   ins_cost(1800);
 10668   format %{ "DSIN   $dst" %}
 10669   opcode(0xD9, 0xFE);
 10670   ins_encode( OpcP, OpcS );
 10671   ins_pipe( pipe_slow );
 10672 %}
 10674 instruct sinXD_reg(regXD dst, eFlagsReg cr) %{
 10675   predicate (UseSSE>=2);
 10676   match(Set dst (SinD dst));
 10677   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
 10678   ins_cost(1800);
 10679   format %{ "DSIN   $dst" %}
 10680   opcode(0xD9, 0xFE);
 10681   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
 10682   ins_pipe( pipe_slow );
 10683 %}
 10685 instruct cosD_reg(regDPR1 dst, regDPR1 src) %{
 10686   predicate (UseSSE<=1);
 10687   match(Set dst (CosD src));
 10688   ins_cost(1800);
 10689   format %{ "DCOS   $dst" %}
 10690   opcode(0xD9, 0xFF);
 10691   ins_encode( OpcP, OpcS );
 10692   ins_pipe( pipe_slow );
 10693 %}
 10695 instruct cosXD_reg(regXD dst, eFlagsReg cr) %{
 10696   predicate (UseSSE>=2);
 10697   match(Set dst (CosD dst));
 10698   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
 10699   ins_cost(1800);
 10700   format %{ "DCOS   $dst" %}
 10701   opcode(0xD9, 0xFF);
 10702   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
 10703   ins_pipe( pipe_slow );
 10704 %}
 10706 instruct tanD_reg(regDPR1 dst, regDPR1 src) %{
 10707   predicate (UseSSE<=1);
 10708   match(Set dst(TanD src));
 10709   format %{ "DTAN   $dst" %}
 10710   ins_encode( Opcode(0xD9), Opcode(0xF2),    // fptan
 10711               Opcode(0xDD), Opcode(0xD8));   // fstp st
 10712   ins_pipe( pipe_slow );
 10713 %}
 10715 instruct tanXD_reg(regXD dst, eFlagsReg cr) %{
 10716   predicate (UseSSE>=2);
 10717   match(Set dst(TanD dst));
 10718   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
 10719   format %{ "DTAN   $dst" %}
 10720   ins_encode( Push_SrcXD(dst),
 10721               Opcode(0xD9), Opcode(0xF2),    // fptan
 10722               Opcode(0xDD), Opcode(0xD8),   // fstp st
 10723               Push_ResultXD(dst) );
 10724   ins_pipe( pipe_slow );
 10725 %}
 10727 instruct atanD_reg(regD dst, regD src) %{
 10728   predicate (UseSSE<=1);
 10729   match(Set dst(AtanD dst src));
 10730   format %{ "DATA   $dst,$src" %}
 10731   opcode(0xD9, 0xF3);
 10732   ins_encode( Push_Reg_D(src),
 10733               OpcP, OpcS, RegOpc(dst) );
 10734   ins_pipe( pipe_slow );
 10735 %}
 10737 instruct atanXD_reg(regXD dst, regXD src, eFlagsReg cr) %{
 10738   predicate (UseSSE>=2);
 10739   match(Set dst(AtanD dst src));
 10740   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
 10741   format %{ "DATA   $dst,$src" %}
 10742   opcode(0xD9, 0xF3);
 10743   ins_encode( Push_SrcXD(src),
 10744               OpcP, OpcS, Push_ResultXD(dst) );
 10745   ins_pipe( pipe_slow );
 10746 %}
 10748 instruct sqrtD_reg(regD dst, regD src) %{
 10749   predicate (UseSSE<=1);
 10750   match(Set dst (SqrtD src));
 10751   format %{ "DSQRT  $dst,$src" %}
 10752   opcode(0xFA, 0xD9);
 10753   ins_encode( Push_Reg_D(src),
 10754               OpcS, OpcP, Pop_Reg_D(dst) );
 10755   ins_pipe( pipe_slow );
 10756 %}
 10758 instruct powD_reg(regD X, regDPR1 Y, eAXRegI rax, eBXRegI rbx, eCXRegI rcx) %{
 10759   predicate (UseSSE<=1);
 10760   match(Set Y (PowD X Y));  // Raise X to the Yth power
 10761   effect(KILL rax, KILL rbx, KILL rcx);
 10762   format %{ "SUB    ESP,8\t\t# Fast-path POW encoding\n\t"
 10763             "FLD_D  $X\n\t"
 10764             "FYL2X  \t\t\t# Q=Y*ln2(X)\n\t"
 10766             "FDUP   \t\t\t# Q Q\n\t"
 10767             "FRNDINT\t\t\t# int(Q) Q\n\t"
 10768             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
 10769             "FISTP  dword [ESP]\n\t"
 10770             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
 10771             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
 10772             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
 10773             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
 10774             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
 10775             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
 10776             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
 10777             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
 10778             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
 10779             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
 10780             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
 10781             "MOV    [ESP+0],0\n\t"
 10782             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
 10784             "ADD    ESP,8"
 10785              %}
 10786   ins_encode( push_stack_temp_qword,
 10787               Push_Reg_D(X),
 10788               Opcode(0xD9), Opcode(0xF1),   // fyl2x
 10789               pow_exp_core_encoding,
 10790               pop_stack_temp_qword);
 10791   ins_pipe( pipe_slow );
 10792 %}
 10794 instruct powXD_reg(regXD dst, regXD src0, regXD src1, regDPR1 tmp1, eAXRegI rax, eBXRegI rbx, eCXRegI rcx ) %{
 10795   predicate (UseSSE>=2);
 10796   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
 10797   effect(KILL tmp1, KILL rax, KILL rbx, KILL rcx );
 10798   format %{ "SUB    ESP,8\t\t# Fast-path POW encoding\n\t"
 10799             "MOVSD  [ESP],$src1\n\t"
 10800             "FLD    FPR1,$src1\n\t"
 10801             "MOVSD  [ESP],$src0\n\t"
 10802             "FLD    FPR1,$src0\n\t"
 10803             "FYL2X  \t\t\t# Q=Y*ln2(X)\n\t"
 10805             "FDUP   \t\t\t# Q Q\n\t"
 10806             "FRNDINT\t\t\t# int(Q) Q\n\t"
 10807             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
 10808             "FISTP  dword [ESP]\n\t"
 10809             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
 10810             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
 10811             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
 10812             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
 10813             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
 10814             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
 10815             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
 10816             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
 10817             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
 10818             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
 10819             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
 10820             "MOV    [ESP+0],0\n\t"
 10821             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
 10823             "FST_D  [ESP]\n\t"
 10824             "MOVSD  $dst,[ESP]\n\t"
 10825             "ADD    ESP,8"
 10826              %}
 10827   ins_encode( push_stack_temp_qword,
 10828               push_xmm_to_fpr1(src1),
 10829               push_xmm_to_fpr1(src0),
 10830               Opcode(0xD9), Opcode(0xF1),   // fyl2x
 10831               pow_exp_core_encoding,
 10832               Push_ResultXD(dst) );
 10833   ins_pipe( pipe_slow );
 10834 %}
 10837 instruct expD_reg(regDPR1 dpr1, eAXRegI rax, eBXRegI rbx, eCXRegI rcx) %{
 10838   predicate (UseSSE<=1);
 10839   match(Set dpr1 (ExpD dpr1));
 10840   effect(KILL rax, KILL rbx, KILL rcx);
 10841   format %{ "SUB    ESP,8\t\t# Fast-path EXP encoding"
 10842             "FLDL2E \t\t\t# Ld log2(e) X\n\t"
 10843             "FMULP  \t\t\t# Q=X*log2(e)\n\t"
 10845             "FDUP   \t\t\t# Q Q\n\t"
 10846             "FRNDINT\t\t\t# int(Q) Q\n\t"
 10847             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
 10848             "FISTP  dword [ESP]\n\t"
 10849             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
 10850             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
 10851             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
 10852             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
 10853             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
 10854             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
 10855             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
 10856             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
 10857             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
 10858             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
 10859             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
 10860             "MOV    [ESP+0],0\n\t"
 10861             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
 10863             "ADD    ESP,8"
 10864              %}
 10865   ins_encode( push_stack_temp_qword,
 10866               Opcode(0xD9), Opcode(0xEA),   // fldl2e
 10867               Opcode(0xDE), Opcode(0xC9),   // fmulp
 10868               pow_exp_core_encoding,
 10869               pop_stack_temp_qword);
 10870   ins_pipe( pipe_slow );
 10871 %}
 10873 instruct expXD_reg(regXD dst, regXD src, regDPR1 tmp1, eAXRegI rax, eBXRegI rbx, eCXRegI rcx) %{
 10874   predicate (UseSSE>=2);
 10875   match(Set dst (ExpD src));
 10876   effect(KILL tmp1, KILL rax, KILL rbx, KILL rcx);
 10877   format %{ "SUB    ESP,8\t\t# Fast-path EXP encoding\n\t"
 10878             "MOVSD  [ESP],$src\n\t"
 10879             "FLDL2E \t\t\t# Ld log2(e) X\n\t"
 10880             "FMULP  \t\t\t# Q=X*log2(e) X\n\t"
 10882             "FDUP   \t\t\t# Q Q\n\t"
 10883             "FRNDINT\t\t\t# int(Q) Q\n\t"
 10884             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
 10885             "FISTP  dword [ESP]\n\t"
 10886             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
 10887             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
 10888             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
 10889             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
 10890             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
 10891             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
 10892             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
 10893             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
 10894             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
 10895             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
 10896             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
 10897             "MOV    [ESP+0],0\n\t"
 10898             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
 10900             "FST_D  [ESP]\n\t"
 10901             "MOVSD  $dst,[ESP]\n\t"
 10902             "ADD    ESP,8"
 10903              %}
 10904   ins_encode( Push_SrcXD(src),
 10905               Opcode(0xD9), Opcode(0xEA),   // fldl2e
 10906               Opcode(0xDE), Opcode(0xC9),   // fmulp
 10907               pow_exp_core_encoding,
 10908               Push_ResultXD(dst) );
 10909   ins_pipe( pipe_slow );
 10910 %}
 10914 instruct log10D_reg(regDPR1 dst, regDPR1 src) %{
 10915   predicate (UseSSE<=1);
 10916   // The source Double operand on FPU stack
 10917   match(Set dst (Log10D src));
 10918   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
 10919   // fxch         ; swap ST(0) with ST(1)
 10920   // fyl2x        ; compute log_10(2) * log_2(x)
 10921   format %{ "FLDLG2 \t\t\t#Log10\n\t"
 10922             "FXCH   \n\t"
 10923             "FYL2X  \t\t\t# Q=Log10*Log_2(x)"
 10924          %}
 10925   ins_encode( Opcode(0xD9), Opcode(0xEC),   // fldlg2
 10926               Opcode(0xD9), Opcode(0xC9),   // fxch
 10927               Opcode(0xD9), Opcode(0xF1));  // fyl2x
 10929   ins_pipe( pipe_slow );
 10930 %}
 10932 instruct log10XD_reg(regXD dst, regXD src, eFlagsReg cr) %{
 10933   predicate (UseSSE>=2);
 10934   effect(KILL cr);
 10935   match(Set dst (Log10D src));
 10936   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
 10937   // fyl2x        ; compute log_10(2) * log_2(x)
 10938   format %{ "FLDLG2 \t\t\t#Log10\n\t"
 10939             "FYL2X  \t\t\t# Q=Log10*Log_2(x)"
 10940          %}
 10941   ins_encode( Opcode(0xD9), Opcode(0xEC),   // fldlg2
 10942               Push_SrcXD(src),
 10943               Opcode(0xD9), Opcode(0xF1),   // fyl2x
 10944               Push_ResultXD(dst));
 10946   ins_pipe( pipe_slow );
 10947 %}
 10949 instruct logD_reg(regDPR1 dst, regDPR1 src) %{
 10950   predicate (UseSSE<=1);
 10951   // The source Double operand on FPU stack
 10952   match(Set dst (LogD src));
 10953   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
 10954   // fxch         ; swap ST(0) with ST(1)
 10955   // fyl2x        ; compute log_e(2) * log_2(x)
 10956   format %{ "FLDLN2 \t\t\t#Log_e\n\t"
 10957             "FXCH   \n\t"
 10958             "FYL2X  \t\t\t# Q=Log_e*Log_2(x)"
 10959          %}
 10960   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
 10961               Opcode(0xD9), Opcode(0xC9),   // fxch
 10962               Opcode(0xD9), Opcode(0xF1));  // fyl2x
 10964   ins_pipe( pipe_slow );
 10965 %}
 10967 instruct logXD_reg(regXD dst, regXD src, eFlagsReg cr) %{
 10968   predicate (UseSSE>=2);
 10969   effect(KILL cr);
 10970   // The source and result Double operands in XMM registers
 10971   match(Set dst (LogD src));
 10972   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
 10973   // fyl2x        ; compute log_e(2) * log_2(x)
 10974   format %{ "FLDLN2 \t\t\t#Log_e\n\t"
 10975             "FYL2X  \t\t\t# Q=Log_e*Log_2(x)"
 10976          %}
 10977   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
 10978               Push_SrcXD(src),
 10979               Opcode(0xD9), Opcode(0xF1),   // fyl2x
 10980               Push_ResultXD(dst));
 10981   ins_pipe( pipe_slow );
 10982 %}
 10984 //-------------Float Instructions-------------------------------
 10985 // Float Math
 10987 // Code for float compare:
 10988 //     fcompp();
 10989 //     fwait(); fnstsw_ax();
 10990 //     sahf();
 10991 //     movl(dst, unordered_result);
 10992 //     jcc(Assembler::parity, exit);
 10993 //     movl(dst, less_result);
 10994 //     jcc(Assembler::below, exit);
 10995 //     movl(dst, equal_result);
 10996 //     jcc(Assembler::equal, exit);
 10997 //     movl(dst, greater_result);
 10998 //   exit:
 11000 // P6 version of float compare, sets condition codes in EFLAGS
 11001 instruct cmpF_cc_P6(eFlagsRegU cr, regF src1, regF src2, eAXRegI rax) %{
 11002   predicate(VM_Version::supports_cmov() && UseSSE == 0);
 11003   match(Set cr (CmpF src1 src2));
 11004   effect(KILL rax);
 11005   ins_cost(150);
 11006   format %{ "FLD    $src1\n\t"
 11007             "FUCOMIP ST,$src2  // P6 instruction\n\t"
 11008             "JNP    exit\n\t"
 11009             "MOV    ah,1       // saw a NaN, set CF (treat as LT)\n\t"
 11010             "SAHF\n"
 11011      "exit:\tNOP               // avoid branch to branch" %}
 11012   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
 11013   ins_encode( Push_Reg_D(src1),
 11014               OpcP, RegOpc(src2),
 11015               cmpF_P6_fixup );
 11016   ins_pipe( pipe_slow );
 11017 %}
 11019 instruct cmpF_cc_P6CF(eFlagsRegUCF cr, regF src1, regF src2) %{
 11020   predicate(VM_Version::supports_cmov() && UseSSE == 0);
 11021   match(Set cr (CmpF src1 src2));
 11022   ins_cost(100);
 11023   format %{ "FLD    $src1\n\t"
 11024             "FUCOMIP ST,$src2  // P6 instruction" %}
 11025   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
 11026   ins_encode( Push_Reg_D(src1),
 11027               OpcP, RegOpc(src2));
 11028   ins_pipe( pipe_slow );
 11029 %}
 11032 // Compare & branch
 11033 instruct cmpF_cc(eFlagsRegU cr, regF src1, regF src2, eAXRegI rax) %{
 11034   predicate(UseSSE == 0);
 11035   match(Set cr (CmpF src1 src2));
 11036   effect(KILL rax);
 11037   ins_cost(200);
 11038   format %{ "FLD    $src1\n\t"
 11039             "FCOMp  $src2\n\t"
 11040             "FNSTSW AX\n\t"
 11041             "TEST   AX,0x400\n\t"
 11042             "JZ,s   flags\n\t"
 11043             "MOV    AH,1\t# unordered treat as LT\n"
 11044     "flags:\tSAHF" %}
 11045   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
 11046   ins_encode( Push_Reg_D(src1),
 11047               OpcP, RegOpc(src2),
 11048               fpu_flags);
 11049   ins_pipe( pipe_slow );
 11050 %}
 11052 // Compare vs zero into -1,0,1
 11053 instruct cmpF_0(eRegI dst, regF src1, immF0 zero, eAXRegI rax, eFlagsReg cr) %{
 11054   predicate(UseSSE == 0);
 11055   match(Set dst (CmpF3 src1 zero));
 11056   effect(KILL cr, KILL rax);
 11057   ins_cost(280);
 11058   format %{ "FTSTF  $dst,$src1" %}
 11059   opcode(0xE4, 0xD9);
 11060   ins_encode( Push_Reg_D(src1),
 11061               OpcS, OpcP, PopFPU,
 11062               CmpF_Result(dst));
 11063   ins_pipe( pipe_slow );
 11064 %}
 11066 // Compare into -1,0,1
 11067 instruct cmpF_reg(eRegI dst, regF src1, regF src2, eAXRegI rax, eFlagsReg cr) %{
 11068   predicate(UseSSE == 0);
 11069   match(Set dst (CmpF3 src1 src2));
 11070   effect(KILL cr, KILL rax);
 11071   ins_cost(300);
 11072   format %{ "FCMPF  $dst,$src1,$src2" %}
 11073   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
 11074   ins_encode( Push_Reg_D(src1),
 11075               OpcP, RegOpc(src2),
 11076               CmpF_Result(dst));
 11077   ins_pipe( pipe_slow );
 11078 %}
 11080 // float compare and set condition codes in EFLAGS by XMM regs
 11081 instruct cmpX_cc(eFlagsRegU cr, regX dst, regX src, eAXRegI rax) %{
 11082   predicate(UseSSE>=1);
 11083   match(Set cr (CmpF dst src));
 11084   effect(KILL rax);
 11085   ins_cost(145);
 11086   format %{ "COMISS $dst,$src\n"
 11087           "\tJNP    exit\n"
 11088           "\tMOV    ah,1       // saw a NaN, set CF\n"
 11089           "\tSAHF\n"
 11090      "exit:\tNOP               // avoid branch to branch" %}
 11091   opcode(0x0F, 0x2F);
 11092   ins_encode(OpcP, OpcS, RegReg(dst, src), cmpF_P6_fixup);
 11093   ins_pipe( pipe_slow );
 11094 %}
 11096 instruct cmpX_ccCF(eFlagsRegUCF cr, regX dst, regX src) %{
 11097   predicate(UseSSE>=1);
 11098   match(Set cr (CmpF dst src));
 11099   ins_cost(100);
 11100   format %{ "COMISS $dst,$src" %}
 11101   opcode(0x0F, 0x2F);
 11102   ins_encode(OpcP, OpcS, RegReg(dst, src));
 11103   ins_pipe( pipe_slow );
 11104 %}
 11106 // float compare and set condition codes in EFLAGS by XMM regs
 11107 instruct cmpX_ccmem(eFlagsRegU cr, regX dst, memory src, eAXRegI rax) %{
 11108   predicate(UseSSE>=1);
 11109   match(Set cr (CmpF dst (LoadF src)));
 11110   effect(KILL rax);
 11111   ins_cost(165);
 11112   format %{ "COMISS $dst,$src\n"
 11113           "\tJNP    exit\n"
 11114           "\tMOV    ah,1       // saw a NaN, set CF\n"
 11115           "\tSAHF\n"
 11116      "exit:\tNOP               // avoid branch to branch" %}
 11117   opcode(0x0F, 0x2F);
 11118   ins_encode(OpcP, OpcS, RegMem(dst, src), cmpF_P6_fixup);
 11119   ins_pipe( pipe_slow );
 11120 %}
 11122 instruct cmpX_ccmemCF(eFlagsRegUCF cr, regX dst, memory src) %{
 11123   predicate(UseSSE>=1);
 11124   match(Set cr (CmpF dst (LoadF src)));
 11125   ins_cost(100);
 11126   format %{ "COMISS $dst,$src" %}
 11127   opcode(0x0F, 0x2F);
 11128   ins_encode(OpcP, OpcS, RegMem(dst, src));
 11129   ins_pipe( pipe_slow );
 11130 %}
 11132 // Compare into -1,0,1 in XMM
 11133 instruct cmpX_reg(eRegI dst, regX src1, regX src2, eFlagsReg cr) %{
 11134   predicate(UseSSE>=1);
 11135   match(Set dst (CmpF3 src1 src2));
 11136   effect(KILL cr);
 11137   ins_cost(255);
 11138   format %{ "XOR    $dst,$dst\n"
 11139           "\tCOMISS $src1,$src2\n"
 11140           "\tJP,s   nan\n"
 11141           "\tJEQ,s  exit\n"
 11142           "\tJA,s   inc\n"
 11143       "nan:\tDEC    $dst\n"
 11144           "\tJMP,s  exit\n"
 11145       "inc:\tINC    $dst\n"
 11146       "exit:"
 11147                 %}
 11148   opcode(0x0F, 0x2F);
 11149   ins_encode(Xor_Reg(dst), OpcP, OpcS, RegReg(src1, src2), CmpX_Result(dst));
 11150   ins_pipe( pipe_slow );
 11151 %}
 11153 // Compare into -1,0,1 in XMM and memory
 11154 instruct cmpX_regmem(eRegI dst, regX src1, memory mem, eFlagsReg cr) %{
 11155   predicate(UseSSE>=1);
 11156   match(Set dst (CmpF3 src1 (LoadF mem)));
 11157   effect(KILL cr);
 11158   ins_cost(275);
 11159   format %{ "COMISS $src1,$mem\n"
 11160           "\tMOV    $dst,0\t\t# do not blow flags\n"
 11161           "\tJP,s   nan\n"
 11162           "\tJEQ,s  exit\n"
 11163           "\tJA,s   inc\n"
 11164       "nan:\tDEC    $dst\n"
 11165           "\tJMP,s  exit\n"
 11166       "inc:\tINC    $dst\n"
 11167       "exit:"
 11168                 %}
 11169   opcode(0x0F, 0x2F);
 11170   ins_encode(OpcP, OpcS, RegMem(src1, mem), LdImmI(dst,0x0), CmpX_Result(dst));
 11171   ins_pipe( pipe_slow );
 11172 %}
 11174 // Spill to obtain 24-bit precision
 11175 instruct subF24_reg(stackSlotF dst, regF src1, regF src2) %{
 11176   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11177   match(Set dst (SubF src1 src2));
 11179   format %{ "FSUB   $dst,$src1 - $src2" %}
 11180   opcode(0xD8, 0x4); /* D8 E0+i or D8 /4 mod==0x3 ;; result in TOS */
 11181   ins_encode( Push_Reg_F(src1),
 11182               OpcReg_F(src2),
 11183               Pop_Mem_F(dst) );
 11184   ins_pipe( fpu_mem_reg_reg );
 11185 %}
 11186 //
 11187 // This instruction does not round to 24-bits
 11188 instruct subF_reg(regF dst, regF src) %{
 11189   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11190   match(Set dst (SubF dst src));
 11192   format %{ "FSUB   $dst,$src" %}
 11193   opcode(0xDE, 0x5); /* DE E8+i  or DE /5 */
 11194   ins_encode( Push_Reg_F(src),
 11195               OpcP, RegOpc(dst) );
 11196   ins_pipe( fpu_reg_reg );
 11197 %}
 11199 // Spill to obtain 24-bit precision
 11200 instruct addF24_reg(stackSlotF dst, regF src1, regF src2) %{
 11201   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11202   match(Set dst (AddF src1 src2));
 11204   format %{ "FADD   $dst,$src1,$src2" %}
 11205   opcode(0xD8, 0x0); /* D8 C0+i */
 11206   ins_encode( Push_Reg_F(src2),
 11207               OpcReg_F(src1),
 11208               Pop_Mem_F(dst) );
 11209   ins_pipe( fpu_mem_reg_reg );
 11210 %}
 11211 //
 11212 // This instruction does not round to 24-bits
 11213 instruct addF_reg(regF dst, regF src) %{
 11214   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11215   match(Set dst (AddF dst src));
 11217   format %{ "FLD    $src\n\t"
 11218             "FADDp  $dst,ST" %}
 11219   opcode(0xDE, 0x0); /* DE C0+i or DE /0*/
 11220   ins_encode( Push_Reg_F(src),
 11221               OpcP, RegOpc(dst) );
 11222   ins_pipe( fpu_reg_reg );
 11223 %}
 11225 // Add two single precision floating point values in xmm
 11226 instruct addX_reg(regX dst, regX src) %{
 11227   predicate(UseSSE>=1);
 11228   match(Set dst (AddF dst src));
 11229   format %{ "ADDSS  $dst,$src" %}
 11230   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x58), RegReg(dst, src));
 11231   ins_pipe( pipe_slow );
 11232 %}
 11234 instruct addX_imm(regX dst, immXF con) %{
 11235   predicate(UseSSE>=1);
 11236   match(Set dst (AddF dst con));
 11237   format %{ "ADDSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
 11238   ins_encode %{
 11239     __ addss($dst$$XMMRegister, $constantaddress($con));
 11240   %}
 11241   ins_pipe(pipe_slow);
 11242 %}
 11244 instruct addX_mem(regX dst, memory mem) %{
 11245   predicate(UseSSE>=1);
 11246   match(Set dst (AddF dst (LoadF mem)));
 11247   format %{ "ADDSS  $dst,$mem" %}
 11248   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x58), RegMem(dst, mem));
 11249   ins_pipe( pipe_slow );
 11250 %}
 11252 // Subtract two single precision floating point values in xmm
 11253 instruct subX_reg(regX dst, regX src) %{
 11254   predicate(UseSSE>=1);
 11255   match(Set dst (SubF dst src));
 11256   format %{ "SUBSS  $dst,$src" %}
 11257   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5C), RegReg(dst, src));
 11258   ins_pipe( pipe_slow );
 11259 %}
 11261 instruct subX_imm(regX dst, immXF con) %{
 11262   predicate(UseSSE>=1);
 11263   match(Set dst (SubF dst con));
 11264   format %{ "SUBSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
 11265   ins_encode %{
 11266     __ subss($dst$$XMMRegister, $constantaddress($con));
 11267   %}
 11268   ins_pipe(pipe_slow);
 11269 %}
 11271 instruct subX_mem(regX dst, memory mem) %{
 11272   predicate(UseSSE>=1);
 11273   match(Set dst (SubF dst (LoadF mem)));
 11274   format %{ "SUBSS  $dst,$mem" %}
 11275   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5C), RegMem(dst,mem));
 11276   ins_pipe( pipe_slow );
 11277 %}
 11279 // Multiply two single precision floating point values in xmm
 11280 instruct mulX_reg(regX dst, regX src) %{
 11281   predicate(UseSSE>=1);
 11282   match(Set dst (MulF dst src));
 11283   format %{ "MULSS  $dst,$src" %}
 11284   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x59), RegReg(dst, src));
 11285   ins_pipe( pipe_slow );
 11286 %}
 11288 instruct mulX_imm(regX dst, immXF con) %{
 11289   predicate(UseSSE>=1);
 11290   match(Set dst (MulF dst con));
 11291   format %{ "MULSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
 11292   ins_encode %{
 11293     __ mulss($dst$$XMMRegister, $constantaddress($con));
 11294   %}
 11295   ins_pipe(pipe_slow);
 11296 %}
 11298 instruct mulX_mem(regX dst, memory mem) %{
 11299   predicate(UseSSE>=1);
 11300   match(Set dst (MulF dst (LoadF mem)));
 11301   format %{ "MULSS  $dst,$mem" %}
 11302   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x59), RegMem(dst,mem));
 11303   ins_pipe( pipe_slow );
 11304 %}
 11306 // Divide two single precision floating point values in xmm
 11307 instruct divX_reg(regX dst, regX src) %{
 11308   predicate(UseSSE>=1);
 11309   match(Set dst (DivF dst src));
 11310   format %{ "DIVSS  $dst,$src" %}
 11311   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5E), RegReg(dst, src));
 11312   ins_pipe( pipe_slow );
 11313 %}
 11315 instruct divX_imm(regX dst, immXF con) %{
 11316   predicate(UseSSE>=1);
 11317   match(Set dst (DivF dst con));
 11318   format %{ "DIVSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
 11319   ins_encode %{
 11320     __ divss($dst$$XMMRegister, $constantaddress($con));
 11321   %}
 11322   ins_pipe(pipe_slow);
 11323 %}
 11325 instruct divX_mem(regX dst, memory mem) %{
 11326   predicate(UseSSE>=1);
 11327   match(Set dst (DivF dst (LoadF mem)));
 11328   format %{ "DIVSS  $dst,$mem" %}
 11329   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5E), RegMem(dst,mem));
 11330   ins_pipe( pipe_slow );
 11331 %}
 11333 // Get the square root of a single precision floating point values in xmm
 11334 instruct sqrtX_reg(regX dst, regX src) %{
 11335   predicate(UseSSE>=1);
 11336   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
 11337   format %{ "SQRTSS $dst,$src" %}
 11338   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x51), RegReg(dst, src));
 11339   ins_pipe( pipe_slow );
 11340 %}
 11342 instruct sqrtX_mem(regX dst, memory mem) %{
 11343   predicate(UseSSE>=1);
 11344   match(Set dst (ConvD2F (SqrtD (ConvF2D (LoadF mem)))));
 11345   format %{ "SQRTSS $dst,$mem" %}
 11346   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x51), RegMem(dst, mem));
 11347   ins_pipe( pipe_slow );
 11348 %}
 11350 // Get the square root of a double precision floating point values in xmm
 11351 instruct sqrtXD_reg(regXD dst, regXD src) %{
 11352   predicate(UseSSE>=2);
 11353   match(Set dst (SqrtD src));
 11354   format %{ "SQRTSD $dst,$src" %}
 11355   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x51), RegReg(dst, src));
 11356   ins_pipe( pipe_slow );
 11357 %}
 11359 instruct sqrtXD_mem(regXD dst, memory mem) %{
 11360   predicate(UseSSE>=2);
 11361   match(Set dst (SqrtD (LoadD mem)));
 11362   format %{ "SQRTSD $dst,$mem" %}
 11363   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x51), RegMem(dst, mem));
 11364   ins_pipe( pipe_slow );
 11365 %}
 11367 instruct absF_reg(regFPR1 dst, regFPR1 src) %{
 11368   predicate(UseSSE==0);
 11369   match(Set dst (AbsF src));
 11370   ins_cost(100);
 11371   format %{ "FABS" %}
 11372   opcode(0xE1, 0xD9);
 11373   ins_encode( OpcS, OpcP );
 11374   ins_pipe( fpu_reg_reg );
 11375 %}
 11377 instruct absX_reg(regX dst ) %{
 11378   predicate(UseSSE>=1);
 11379   match(Set dst (AbsF dst));
 11380   format %{ "ANDPS  $dst,[0x7FFFFFFF]\t# ABS F by sign masking" %}
 11381   ins_encode( AbsXF_encoding(dst));
 11382   ins_pipe( pipe_slow );
 11383 %}
 11385 instruct negF_reg(regFPR1 dst, regFPR1 src) %{
 11386   predicate(UseSSE==0);
 11387   match(Set dst (NegF src));
 11388   ins_cost(100);
 11389   format %{ "FCHS" %}
 11390   opcode(0xE0, 0xD9);
 11391   ins_encode( OpcS, OpcP );
 11392   ins_pipe( fpu_reg_reg );
 11393 %}
 11395 instruct negX_reg( regX dst ) %{
 11396   predicate(UseSSE>=1);
 11397   match(Set dst (NegF dst));
 11398   format %{ "XORPS  $dst,[0x80000000]\t# CHS F by sign flipping" %}
 11399   ins_encode( NegXF_encoding(dst));
 11400   ins_pipe( pipe_slow );
 11401 %}
 11403 // Cisc-alternate to addF_reg
 11404 // Spill to obtain 24-bit precision
 11405 instruct addF24_reg_mem(stackSlotF dst, regF src1, memory src2) %{
 11406   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11407   match(Set dst (AddF src1 (LoadF src2)));
 11409   format %{ "FLD    $src2\n\t"
 11410             "FADD   ST,$src1\n\t"
 11411             "FSTP_S $dst" %}
 11412   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
 11413   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
 11414               OpcReg_F(src1),
 11415               Pop_Mem_F(dst) );
 11416   ins_pipe( fpu_mem_reg_mem );
 11417 %}
 11418 //
 11419 // Cisc-alternate to addF_reg
 11420 // This instruction does not round to 24-bits
 11421 instruct addF_reg_mem(regF dst, memory src) %{
 11422   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11423   match(Set dst (AddF dst (LoadF src)));
 11425   format %{ "FADD   $dst,$src" %}
 11426   opcode(0xDE, 0x0, 0xD9); /* DE C0+i or DE /0*/  /* LoadF  D9 /0 */
 11427   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
 11428               OpcP, RegOpc(dst) );
 11429   ins_pipe( fpu_reg_mem );
 11430 %}
 11432 // // Following two instructions for _222_mpegaudio
 11433 // Spill to obtain 24-bit precision
 11434 instruct addF24_mem_reg(stackSlotF dst, regF src2, memory src1 ) %{
 11435   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11436   match(Set dst (AddF src1 src2));
 11438   format %{ "FADD   $dst,$src1,$src2" %}
 11439   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
 11440   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src1),
 11441               OpcReg_F(src2),
 11442               Pop_Mem_F(dst) );
 11443   ins_pipe( fpu_mem_reg_mem );
 11444 %}
 11446 // Cisc-spill variant
 11447 // Spill to obtain 24-bit precision
 11448 instruct addF24_mem_cisc(stackSlotF dst, memory src1, memory src2) %{
 11449   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11450   match(Set dst (AddF src1 (LoadF src2)));
 11452   format %{ "FADD   $dst,$src1,$src2 cisc" %}
 11453   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
 11454   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
 11455               set_instruction_start,
 11456               OpcP, RMopc_Mem(secondary,src1),
 11457               Pop_Mem_F(dst) );
 11458   ins_pipe( fpu_mem_mem_mem );
 11459 %}
 11461 // Spill to obtain 24-bit precision
 11462 instruct addF24_mem_mem(stackSlotF dst, memory src1, memory src2) %{
 11463   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11464   match(Set dst (AddF src1 src2));
 11466   format %{ "FADD   $dst,$src1,$src2" %}
 11467   opcode(0xD8, 0x0, 0xD9); /* D8 /0 */  /* LoadF  D9 /0 */
 11468   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
 11469               set_instruction_start,
 11470               OpcP, RMopc_Mem(secondary,src1),
 11471               Pop_Mem_F(dst) );
 11472   ins_pipe( fpu_mem_mem_mem );
 11473 %}
 11476 // Spill to obtain 24-bit precision
 11477 instruct addF24_reg_imm(stackSlotF dst, regF src, immF con) %{
 11478   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11479   match(Set dst (AddF src con));
 11480   format %{ "FLD    $src\n\t"
 11481             "FADD_S [$constantaddress]\t# load from constant table: float=$con\n\t"
 11482             "FSTP_S $dst"  %}
 11483   ins_encode %{
 11484     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
 11485     __ fadd_s($constantaddress($con));
 11486     __ fstp_s(Address(rsp, $dst$$disp));
 11487   %}
 11488   ins_pipe(fpu_mem_reg_con);
 11489 %}
 11490 //
 11491 // This instruction does not round to 24-bits
 11492 instruct addF_reg_imm(regF dst, regF src, immF con) %{
 11493   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11494   match(Set dst (AddF src con));
 11495   format %{ "FLD    $src\n\t"
 11496             "FADD_S [$constantaddress]\t# load from constant table: float=$con\n\t"
 11497             "FSTP   $dst"  %}
 11498   ins_encode %{
 11499     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
 11500     __ fadd_s($constantaddress($con));
 11501     __ fstp_d($dst$$reg);
 11502   %}
 11503   ins_pipe(fpu_reg_reg_con);
 11504 %}
 11506 // Spill to obtain 24-bit precision
 11507 instruct mulF24_reg(stackSlotF dst, regF src1, regF src2) %{
 11508   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11509   match(Set dst (MulF src1 src2));
 11511   format %{ "FLD    $src1\n\t"
 11512             "FMUL   $src2\n\t"
 11513             "FSTP_S $dst"  %}
 11514   opcode(0xD8, 0x1); /* D8 C8+i or D8 /1 ;; result in TOS */
 11515   ins_encode( Push_Reg_F(src1),
 11516               OpcReg_F(src2),
 11517               Pop_Mem_F(dst) );
 11518   ins_pipe( fpu_mem_reg_reg );
 11519 %}
 11520 //
 11521 // This instruction does not round to 24-bits
 11522 instruct mulF_reg(regF dst, regF src1, regF src2) %{
 11523   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11524   match(Set dst (MulF src1 src2));
 11526   format %{ "FLD    $src1\n\t"
 11527             "FMUL   $src2\n\t"
 11528             "FSTP_S $dst"  %}
 11529   opcode(0xD8, 0x1); /* D8 C8+i */
 11530   ins_encode( Push_Reg_F(src2),
 11531               OpcReg_F(src1),
 11532               Pop_Reg_F(dst) );
 11533   ins_pipe( fpu_reg_reg_reg );
 11534 %}
 11537 // Spill to obtain 24-bit precision
 11538 // Cisc-alternate to reg-reg multiply
 11539 instruct mulF24_reg_mem(stackSlotF dst, regF src1, memory src2) %{
 11540   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11541   match(Set dst (MulF src1 (LoadF src2)));
 11543   format %{ "FLD_S  $src2\n\t"
 11544             "FMUL   $src1\n\t"
 11545             "FSTP_S $dst"  %}
 11546   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i or DE /1*/  /* LoadF D9 /0 */
 11547   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
 11548               OpcReg_F(src1),
 11549               Pop_Mem_F(dst) );
 11550   ins_pipe( fpu_mem_reg_mem );
 11551 %}
 11552 //
 11553 // This instruction does not round to 24-bits
 11554 // Cisc-alternate to reg-reg multiply
 11555 instruct mulF_reg_mem(regF dst, regF src1, memory src2) %{
 11556   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11557   match(Set dst (MulF src1 (LoadF src2)));
 11559   format %{ "FMUL   $dst,$src1,$src2" %}
 11560   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i */  /* LoadF D9 /0 */
 11561   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
 11562               OpcReg_F(src1),
 11563               Pop_Reg_F(dst) );
 11564   ins_pipe( fpu_reg_reg_mem );
 11565 %}
 11567 // Spill to obtain 24-bit precision
 11568 instruct mulF24_mem_mem(stackSlotF dst, memory src1, memory src2) %{
 11569   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11570   match(Set dst (MulF src1 src2));
 11572   format %{ "FMUL   $dst,$src1,$src2" %}
 11573   opcode(0xD8, 0x1, 0xD9); /* D8 /1 */  /* LoadF D9 /0 */
 11574   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
 11575               set_instruction_start,
 11576               OpcP, RMopc_Mem(secondary,src1),
 11577               Pop_Mem_F(dst) );
 11578   ins_pipe( fpu_mem_mem_mem );
 11579 %}
 11581 // Spill to obtain 24-bit precision
 11582 instruct mulF24_reg_imm(stackSlotF dst, regF src, immF con) %{
 11583   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11584   match(Set dst (MulF src con));
 11586   format %{ "FLD    $src\n\t"
 11587             "FMUL_S [$constantaddress]\t# load from constant table: float=$con\n\t"
 11588             "FSTP_S $dst"  %}
 11589   ins_encode %{
 11590     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
 11591     __ fmul_s($constantaddress($con));
 11592     __ fstp_s(Address(rsp, $dst$$disp));
 11593   %}
 11594   ins_pipe(fpu_mem_reg_con);
 11595 %}
 11596 //
 11597 // This instruction does not round to 24-bits
 11598 instruct mulF_reg_imm(regF dst, regF src, immF con) %{
 11599   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11600   match(Set dst (MulF src con));
 11602   format %{ "FLD    $src\n\t"
 11603             "FMUL_S [$constantaddress]\t# load from constant table: float=$con\n\t"
 11604             "FSTP   $dst"  %}
 11605   ins_encode %{
 11606     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
 11607     __ fmul_s($constantaddress($con));
 11608     __ fstp_d($dst$$reg);
 11609   %}
 11610   ins_pipe(fpu_reg_reg_con);
 11611 %}
 11614 //
 11615 // MACRO1 -- subsume unshared load into mulF
 11616 // This instruction does not round to 24-bits
 11617 instruct mulF_reg_load1(regF dst, regF src, memory mem1 ) %{
 11618   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11619   match(Set dst (MulF (LoadF mem1) src));
 11621   format %{ "FLD    $mem1    ===MACRO1===\n\t"
 11622             "FMUL   ST,$src\n\t"
 11623             "FSTP   $dst" %}
 11624   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i or D8 /1 */  /* LoadF D9 /0 */
 11625   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,mem1),
 11626               OpcReg_F(src),
 11627               Pop_Reg_F(dst) );
 11628   ins_pipe( fpu_reg_reg_mem );
 11629 %}
 11630 //
 11631 // MACRO2 -- addF a mulF which subsumed an unshared load
 11632 // This instruction does not round to 24-bits
 11633 instruct addF_mulF_reg_load1(regF dst, memory mem1, regF src1, regF src2) %{
 11634   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11635   match(Set dst (AddF (MulF (LoadF mem1) src1) src2));
 11636   ins_cost(95);
 11638   format %{ "FLD    $mem1     ===MACRO2===\n\t"
 11639             "FMUL   ST,$src1  subsume mulF left load\n\t"
 11640             "FADD   ST,$src2\n\t"
 11641             "FSTP   $dst" %}
 11642   opcode(0xD9); /* LoadF D9 /0 */
 11643   ins_encode( OpcP, RMopc_Mem(0x00,mem1),
 11644               FMul_ST_reg(src1),
 11645               FAdd_ST_reg(src2),
 11646               Pop_Reg_F(dst) );
 11647   ins_pipe( fpu_reg_mem_reg_reg );
 11648 %}
 11650 // MACRO3 -- addF a mulF
 11651 // This instruction does not round to 24-bits.  It is a '2-address'
 11652 // instruction in that the result goes back to src2.  This eliminates
 11653 // a move from the macro; possibly the register allocator will have
 11654 // to add it back (and maybe not).
 11655 instruct addF_mulF_reg(regF src2, regF src1, regF src0) %{
 11656   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11657   match(Set src2 (AddF (MulF src0 src1) src2));
 11659   format %{ "FLD    $src0     ===MACRO3===\n\t"
 11660             "FMUL   ST,$src1\n\t"
 11661             "FADDP  $src2,ST" %}
 11662   opcode(0xD9); /* LoadF D9 /0 */
 11663   ins_encode( Push_Reg_F(src0),
 11664               FMul_ST_reg(src1),
 11665               FAddP_reg_ST(src2) );
 11666   ins_pipe( fpu_reg_reg_reg );
 11667 %}
 11669 // MACRO4 -- divF subF
 11670 // This instruction does not round to 24-bits
 11671 instruct subF_divF_reg(regF dst, regF src1, regF src2, regF src3) %{
 11672   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11673   match(Set dst (DivF (SubF src2 src1) src3));
 11675   format %{ "FLD    $src2   ===MACRO4===\n\t"
 11676             "FSUB   ST,$src1\n\t"
 11677             "FDIV   ST,$src3\n\t"
 11678             "FSTP  $dst" %}
 11679   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
 11680   ins_encode( Push_Reg_F(src2),
 11681               subF_divF_encode(src1,src3),
 11682               Pop_Reg_F(dst) );
 11683   ins_pipe( fpu_reg_reg_reg_reg );
 11684 %}
 11686 // Spill to obtain 24-bit precision
 11687 instruct divF24_reg(stackSlotF dst, regF src1, regF src2) %{
 11688   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
 11689   match(Set dst (DivF src1 src2));
 11691   format %{ "FDIV   $dst,$src1,$src2" %}
 11692   opcode(0xD8, 0x6); /* D8 F0+i or DE /6*/
 11693   ins_encode( Push_Reg_F(src1),
 11694               OpcReg_F(src2),
 11695               Pop_Mem_F(dst) );
 11696   ins_pipe( fpu_mem_reg_reg );
 11697 %}
 11698 //
 11699 // This instruction does not round to 24-bits
 11700 instruct divF_reg(regF dst, regF src) %{
 11701   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11702   match(Set dst (DivF dst src));
 11704   format %{ "FDIV   $dst,$src" %}
 11705   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
 11706   ins_encode( Push_Reg_F(src),
 11707               OpcP, RegOpc(dst) );
 11708   ins_pipe( fpu_reg_reg );
 11709 %}
 11712 // Spill to obtain 24-bit precision
 11713 instruct modF24_reg(stackSlotF dst, regF src1, regF src2, eAXRegI rax, eFlagsReg cr) %{
 11714   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
 11715   match(Set dst (ModF src1 src2));
 11716   effect(KILL rax, KILL cr); // emitModD() uses EAX and EFLAGS
 11718   format %{ "FMOD   $dst,$src1,$src2" %}
 11719   ins_encode( Push_Reg_Mod_D(src1, src2),
 11720               emitModD(),
 11721               Push_Result_Mod_D(src2),
 11722               Pop_Mem_F(dst));
 11723   ins_pipe( pipe_slow );
 11724 %}
 11725 //
 11726 // This instruction does not round to 24-bits
 11727 instruct modF_reg(regF dst, regF src, eAXRegI rax, eFlagsReg cr) %{
 11728   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
 11729   match(Set dst (ModF dst src));
 11730   effect(KILL rax, KILL cr); // emitModD() uses EAX and EFLAGS
 11732   format %{ "FMOD   $dst,$src" %}
 11733   ins_encode(Push_Reg_Mod_D(dst, src),
 11734               emitModD(),
 11735               Push_Result_Mod_D(src),
 11736               Pop_Reg_F(dst));
 11737   ins_pipe( pipe_slow );
 11738 %}
 11740 instruct modX_reg(regX dst, regX src0, regX src1, eAXRegI rax, eFlagsReg cr) %{
 11741   predicate(UseSSE>=1);
 11742   match(Set dst (ModF src0 src1));
 11743   effect(KILL rax, KILL cr);
 11744   format %{ "SUB    ESP,4\t # FMOD\n"
 11745           "\tMOVSS  [ESP+0],$src1\n"
 11746           "\tFLD_S  [ESP+0]\n"
 11747           "\tMOVSS  [ESP+0],$src0\n"
 11748           "\tFLD_S  [ESP+0]\n"
 11749      "loop:\tFPREM\n"
 11750           "\tFWAIT\n"
 11751           "\tFNSTSW AX\n"
 11752           "\tSAHF\n"
 11753           "\tJP     loop\n"
 11754           "\tFSTP_S [ESP+0]\n"
 11755           "\tMOVSS  $dst,[ESP+0]\n"
 11756           "\tADD    ESP,4\n"
 11757           "\tFSTP   ST0\t # Restore FPU Stack"
 11758     %}
 11759   ins_cost(250);
 11760   ins_encode( Push_ModX_encoding(src0, src1), emitModD(), Push_ResultX(dst,0x4), PopFPU);
 11761   ins_pipe( pipe_slow );
 11762 %}
 11765 //----------Arithmetic Conversion Instructions---------------------------------
 11766 // The conversions operations are all Alpha sorted.  Please keep it that way!
 11768 instruct roundFloat_mem_reg(stackSlotF dst, regF src) %{
 11769   predicate(UseSSE==0);
 11770   match(Set dst (RoundFloat src));
 11771   ins_cost(125);
 11772   format %{ "FST_S  $dst,$src\t# F-round" %}
 11773   ins_encode( Pop_Mem_Reg_F(dst, src) );
 11774   ins_pipe( fpu_mem_reg );
 11775 %}
 11777 instruct roundDouble_mem_reg(stackSlotD dst, regD src) %{
 11778   predicate(UseSSE<=1);
 11779   match(Set dst (RoundDouble src));
 11780   ins_cost(125);
 11781   format %{ "FST_D  $dst,$src\t# D-round" %}
 11782   ins_encode( Pop_Mem_Reg_D(dst, src) );
 11783   ins_pipe( fpu_mem_reg );
 11784 %}
 11786 // Force rounding to 24-bit precision and 6-bit exponent
 11787 instruct convD2F_reg(stackSlotF dst, regD src) %{
 11788   predicate(UseSSE==0);
 11789   match(Set dst (ConvD2F src));
 11790   format %{ "FST_S  $dst,$src\t# F-round" %}
 11791   expand %{
 11792     roundFloat_mem_reg(dst,src);
 11793   %}
 11794 %}
 11796 // Force rounding to 24-bit precision and 6-bit exponent
 11797 instruct convD2X_reg(regX dst, regD src, eFlagsReg cr) %{
 11798   predicate(UseSSE==1);
 11799   match(Set dst (ConvD2F src));
 11800   effect( KILL cr );
 11801   format %{ "SUB    ESP,4\n\t"
 11802             "FST_S  [ESP],$src\t# F-round\n\t"
 11803             "MOVSS  $dst,[ESP]\n\t"
 11804             "ADD ESP,4" %}
 11805   ins_encode( D2X_encoding(dst, src) );
 11806   ins_pipe( pipe_slow );
 11807 %}
 11809 // Force rounding double precision to single precision
 11810 instruct convXD2X_reg(regX dst, regXD src) %{
 11811   predicate(UseSSE>=2);
 11812   match(Set dst (ConvD2F src));
 11813   format %{ "CVTSD2SS $dst,$src\t# F-round" %}
 11814   opcode(0xF2, 0x0F, 0x5A);
 11815   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
 11816   ins_pipe( pipe_slow );
 11817 %}
 11819 instruct convF2D_reg_reg(regD dst, regF src) %{
 11820   predicate(UseSSE==0);
 11821   match(Set dst (ConvF2D src));
 11822   format %{ "FST_S  $dst,$src\t# D-round" %}
 11823   ins_encode( Pop_Reg_Reg_D(dst, src));
 11824   ins_pipe( fpu_reg_reg );
 11825 %}
 11827 instruct convF2D_reg(stackSlotD dst, regF src) %{
 11828   predicate(UseSSE==1);
 11829   match(Set dst (ConvF2D src));
 11830   format %{ "FST_D  $dst,$src\t# D-round" %}
 11831   expand %{
 11832     roundDouble_mem_reg(dst,src);
 11833   %}
 11834 %}
 11836 instruct convX2D_reg(regD dst, regX src, eFlagsReg cr) %{
 11837   predicate(UseSSE==1);
 11838   match(Set dst (ConvF2D src));
 11839   effect( KILL cr );
 11840   format %{ "SUB    ESP,4\n\t"
 11841             "MOVSS  [ESP] $src\n\t"
 11842             "FLD_S  [ESP]\n\t"
 11843             "ADD    ESP,4\n\t"
 11844             "FSTP   $dst\t# D-round" %}
 11845   ins_encode( X2D_encoding(dst, src), Pop_Reg_D(dst));
 11846   ins_pipe( pipe_slow );
 11847 %}
 11849 instruct convX2XD_reg(regXD dst, regX src) %{
 11850   predicate(UseSSE>=2);
 11851   match(Set dst (ConvF2D src));
 11852   format %{ "CVTSS2SD $dst,$src\t# D-round" %}
 11853   opcode(0xF3, 0x0F, 0x5A);
 11854   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
 11855   ins_pipe( pipe_slow );
 11856 %}
 11858 // Convert a double to an int.  If the double is a NAN, stuff a zero in instead.
 11859 instruct convD2I_reg_reg( eAXRegI dst, eDXRegI tmp, regD src, eFlagsReg cr ) %{
 11860   predicate(UseSSE<=1);
 11861   match(Set dst (ConvD2I src));
 11862   effect( KILL tmp, KILL cr );
 11863   format %{ "FLD    $src\t# Convert double to int \n\t"
 11864             "FLDCW  trunc mode\n\t"
 11865             "SUB    ESP,4\n\t"
 11866             "FISTp  [ESP + #0]\n\t"
 11867             "FLDCW  std/24-bit mode\n\t"
 11868             "POP    EAX\n\t"
 11869             "CMP    EAX,0x80000000\n\t"
 11870             "JNE,s  fast\n\t"
 11871             "FLD_D  $src\n\t"
 11872             "CALL   d2i_wrapper\n"
 11873       "fast:" %}
 11874   ins_encode( Push_Reg_D(src), D2I_encoding(src) );
 11875   ins_pipe( pipe_slow );
 11876 %}
 11878 // Convert a double to an int.  If the double is a NAN, stuff a zero in instead.
 11879 instruct convXD2I_reg_reg( eAXRegI dst, eDXRegI tmp, regXD src, eFlagsReg cr ) %{
 11880   predicate(UseSSE>=2);
 11881   match(Set dst (ConvD2I src));
 11882   effect( KILL tmp, KILL cr );
 11883   format %{ "CVTTSD2SI $dst, $src\n\t"
 11884             "CMP    $dst,0x80000000\n\t"
 11885             "JNE,s  fast\n\t"
 11886             "SUB    ESP, 8\n\t"
 11887             "MOVSD  [ESP], $src\n\t"
 11888             "FLD_D  [ESP]\n\t"
 11889             "ADD    ESP, 8\n\t"
 11890             "CALL   d2i_wrapper\n"
 11891       "fast:" %}
 11892   opcode(0x1); // double-precision conversion
 11893   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x2C), FX2I_encoding(src,dst));
 11894   ins_pipe( pipe_slow );
 11895 %}
 11897 instruct convD2L_reg_reg( eADXRegL dst, regD src, eFlagsReg cr ) %{
 11898   predicate(UseSSE<=1);
 11899   match(Set dst (ConvD2L src));
 11900   effect( KILL cr );
 11901   format %{ "FLD    $src\t# Convert double to long\n\t"
 11902             "FLDCW  trunc mode\n\t"
 11903             "SUB    ESP,8\n\t"
 11904             "FISTp  [ESP + #0]\n\t"
 11905             "FLDCW  std/24-bit mode\n\t"
 11906             "POP    EAX\n\t"
 11907             "POP    EDX\n\t"
 11908             "CMP    EDX,0x80000000\n\t"
 11909             "JNE,s  fast\n\t"
 11910             "TEST   EAX,EAX\n\t"
 11911             "JNE,s  fast\n\t"
 11912             "FLD    $src\n\t"
 11913             "CALL   d2l_wrapper\n"
 11914       "fast:" %}
 11915   ins_encode( Push_Reg_D(src),  D2L_encoding(src) );
 11916   ins_pipe( pipe_slow );
 11917 %}
 11919 // XMM lacks a float/double->long conversion, so use the old FPU stack.
 11920 instruct convXD2L_reg_reg( eADXRegL dst, regXD src, eFlagsReg cr ) %{
 11921   predicate (UseSSE>=2);
 11922   match(Set dst (ConvD2L src));
 11923   effect( KILL cr );
 11924   format %{ "SUB    ESP,8\t# Convert double to long\n\t"
 11925             "MOVSD  [ESP],$src\n\t"
 11926             "FLD_D  [ESP]\n\t"
 11927             "FLDCW  trunc mode\n\t"
 11928             "FISTp  [ESP + #0]\n\t"
 11929             "FLDCW  std/24-bit mode\n\t"
 11930             "POP    EAX\n\t"
 11931             "POP    EDX\n\t"
 11932             "CMP    EDX,0x80000000\n\t"
 11933             "JNE,s  fast\n\t"
 11934             "TEST   EAX,EAX\n\t"
 11935             "JNE,s  fast\n\t"
 11936             "SUB    ESP,8\n\t"
 11937             "MOVSD  [ESP],$src\n\t"
 11938             "FLD_D  [ESP]\n\t"
 11939             "CALL   d2l_wrapper\n"
 11940       "fast:" %}
 11941   ins_encode( XD2L_encoding(src) );
 11942   ins_pipe( pipe_slow );
 11943 %}
 11945 // Convert a double to an int.  Java semantics require we do complex
 11946 // manglations in the corner cases.  So we set the rounding mode to
 11947 // 'zero', store the darned double down as an int, and reset the
 11948 // rounding mode to 'nearest'.  The hardware stores a flag value down
 11949 // if we would overflow or converted a NAN; we check for this and
 11950 // and go the slow path if needed.
 11951 instruct convF2I_reg_reg(eAXRegI dst, eDXRegI tmp, regF src, eFlagsReg cr ) %{
 11952   predicate(UseSSE==0);
 11953   match(Set dst (ConvF2I src));
 11954   effect( KILL tmp, KILL cr );
 11955   format %{ "FLD    $src\t# Convert float to int \n\t"
 11956             "FLDCW  trunc mode\n\t"
 11957             "SUB    ESP,4\n\t"
 11958             "FISTp  [ESP + #0]\n\t"
 11959             "FLDCW  std/24-bit mode\n\t"
 11960             "POP    EAX\n\t"
 11961             "CMP    EAX,0x80000000\n\t"
 11962             "JNE,s  fast\n\t"
 11963             "FLD    $src\n\t"
 11964             "CALL   d2i_wrapper\n"
 11965       "fast:" %}
 11966   // D2I_encoding works for F2I
 11967   ins_encode( Push_Reg_F(src), D2I_encoding(src) );
 11968   ins_pipe( pipe_slow );
 11969 %}
 11971 // Convert a float in xmm to an int reg.
 11972 instruct convX2I_reg(eAXRegI dst, eDXRegI tmp, regX src, eFlagsReg cr ) %{
 11973   predicate(UseSSE>=1);
 11974   match(Set dst (ConvF2I src));
 11975   effect( KILL tmp, KILL cr );
 11976   format %{ "CVTTSS2SI $dst, $src\n\t"
 11977             "CMP    $dst,0x80000000\n\t"
 11978             "JNE,s  fast\n\t"
 11979             "SUB    ESP, 4\n\t"
 11980             "MOVSS  [ESP], $src\n\t"
 11981             "FLD    [ESP]\n\t"
 11982             "ADD    ESP, 4\n\t"
 11983             "CALL   d2i_wrapper\n"
 11984       "fast:" %}
 11985   opcode(0x0); // single-precision conversion
 11986   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x2C), FX2I_encoding(src,dst));
 11987   ins_pipe( pipe_slow );
 11988 %}
 11990 instruct convF2L_reg_reg( eADXRegL dst, regF src, eFlagsReg cr ) %{
 11991   predicate(UseSSE==0);
 11992   match(Set dst (ConvF2L src));
 11993   effect( KILL cr );
 11994   format %{ "FLD    $src\t# Convert float to long\n\t"
 11995             "FLDCW  trunc mode\n\t"
 11996             "SUB    ESP,8\n\t"
 11997             "FISTp  [ESP + #0]\n\t"
 11998             "FLDCW  std/24-bit mode\n\t"
 11999             "POP    EAX\n\t"
 12000             "POP    EDX\n\t"
 12001             "CMP    EDX,0x80000000\n\t"
 12002             "JNE,s  fast\n\t"
 12003             "TEST   EAX,EAX\n\t"
 12004             "JNE,s  fast\n\t"
 12005             "FLD    $src\n\t"
 12006             "CALL   d2l_wrapper\n"
 12007       "fast:" %}
 12008   // D2L_encoding works for F2L
 12009   ins_encode( Push_Reg_F(src), D2L_encoding(src) );
 12010   ins_pipe( pipe_slow );
 12011 %}
 12013 // XMM lacks a float/double->long conversion, so use the old FPU stack.
 12014 instruct convX2L_reg_reg( eADXRegL dst, regX src, eFlagsReg cr ) %{
 12015   predicate (UseSSE>=1);
 12016   match(Set dst (ConvF2L src));
 12017   effect( KILL cr );
 12018   format %{ "SUB    ESP,8\t# Convert float to long\n\t"
 12019             "MOVSS  [ESP],$src\n\t"
 12020             "FLD_S  [ESP]\n\t"
 12021             "FLDCW  trunc mode\n\t"
 12022             "FISTp  [ESP + #0]\n\t"
 12023             "FLDCW  std/24-bit mode\n\t"
 12024             "POP    EAX\n\t"
 12025             "POP    EDX\n\t"
 12026             "CMP    EDX,0x80000000\n\t"
 12027             "JNE,s  fast\n\t"
 12028             "TEST   EAX,EAX\n\t"
 12029             "JNE,s  fast\n\t"
 12030             "SUB    ESP,4\t# Convert float to long\n\t"
 12031             "MOVSS  [ESP],$src\n\t"
 12032             "FLD_S  [ESP]\n\t"
 12033             "ADD    ESP,4\n\t"
 12034             "CALL   d2l_wrapper\n"
 12035       "fast:" %}
 12036   ins_encode( X2L_encoding(src) );
 12037   ins_pipe( pipe_slow );
 12038 %}
 12040 instruct convI2D_reg(regD dst, stackSlotI src) %{
 12041   predicate( UseSSE<=1 );
 12042   match(Set dst (ConvI2D src));
 12043   format %{ "FILD   $src\n\t"
 12044             "FSTP   $dst" %}
 12045   opcode(0xDB, 0x0);  /* DB /0 */
 12046   ins_encode(Push_Mem_I(src), Pop_Reg_D(dst));
 12047   ins_pipe( fpu_reg_mem );
 12048 %}
 12050 instruct convI2XD_reg(regXD dst, eRegI src) %{
 12051   predicate( UseSSE>=2 && !UseXmmI2D );
 12052   match(Set dst (ConvI2D src));
 12053   format %{ "CVTSI2SD $dst,$src" %}
 12054   opcode(0xF2, 0x0F, 0x2A);
 12055   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
 12056   ins_pipe( pipe_slow );
 12057 %}
 12059 instruct convI2XD_mem(regXD dst, memory mem) %{
 12060   predicate( UseSSE>=2 );
 12061   match(Set dst (ConvI2D (LoadI mem)));
 12062   format %{ "CVTSI2SD $dst,$mem" %}
 12063   opcode(0xF2, 0x0F, 0x2A);
 12064   ins_encode( OpcP, OpcS, Opcode(tertiary), RegMem(dst, mem));
 12065   ins_pipe( pipe_slow );
 12066 %}
 12068 instruct convXI2XD_reg(regXD dst, eRegI src)
 12069 %{
 12070   predicate( UseSSE>=2 && UseXmmI2D );
 12071   match(Set dst (ConvI2D src));
 12073   format %{ "MOVD  $dst,$src\n\t"
 12074             "CVTDQ2PD $dst,$dst\t# i2d" %}
 12075   ins_encode %{
 12076     __ movdl($dst$$XMMRegister, $src$$Register);
 12077     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
 12078   %}
 12079   ins_pipe(pipe_slow); // XXX
 12080 %}
 12082 instruct convI2D_mem(regD dst, memory mem) %{
 12083   predicate( UseSSE<=1 && !Compile::current()->select_24_bit_instr());
 12084   match(Set dst (ConvI2D (LoadI mem)));
 12085   format %{ "FILD   $mem\n\t"
 12086             "FSTP   $dst" %}
 12087   opcode(0xDB);      /* DB /0 */
 12088   ins_encode( OpcP, RMopc_Mem(0x00,mem),
 12089               Pop_Reg_D(dst));
 12090   ins_pipe( fpu_reg_mem );
 12091 %}
 12093 // Convert a byte to a float; no rounding step needed.
 12094 instruct conv24I2F_reg(regF dst, stackSlotI src) %{
 12095   predicate( UseSSE==0 && n->in(1)->Opcode() == Op_AndI && n->in(1)->in(2)->is_Con() && n->in(1)->in(2)->get_int() == 255 );
 12096   match(Set dst (ConvI2F src));
 12097   format %{ "FILD   $src\n\t"
 12098             "FSTP   $dst" %}
 12100   opcode(0xDB, 0x0);  /* DB /0 */
 12101   ins_encode(Push_Mem_I(src), Pop_Reg_F(dst));
 12102   ins_pipe( fpu_reg_mem );
 12103 %}
 12105 // In 24-bit mode, force exponent rounding by storing back out
 12106 instruct convI2F_SSF(stackSlotF dst, stackSlotI src) %{
 12107   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
 12108   match(Set dst (ConvI2F src));
 12109   ins_cost(200);
 12110   format %{ "FILD   $src\n\t"
 12111             "FSTP_S $dst" %}
 12112   opcode(0xDB, 0x0);  /* DB /0 */
 12113   ins_encode( Push_Mem_I(src),
 12114               Pop_Mem_F(dst));
 12115   ins_pipe( fpu_mem_mem );
 12116 %}
 12118 // In 24-bit mode, force exponent rounding by storing back out
 12119 instruct convI2F_SSF_mem(stackSlotF dst, memory mem) %{
 12120   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
 12121   match(Set dst (ConvI2F (LoadI mem)));
 12122   ins_cost(200);
 12123   format %{ "FILD   $mem\n\t"
 12124             "FSTP_S $dst" %}
 12125   opcode(0xDB);  /* DB /0 */
 12126   ins_encode( OpcP, RMopc_Mem(0x00,mem),
 12127               Pop_Mem_F(dst));
 12128   ins_pipe( fpu_mem_mem );
 12129 %}
 12131 // This instruction does not round to 24-bits
 12132 instruct convI2F_reg(regF dst, stackSlotI src) %{
 12133   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
 12134   match(Set dst (ConvI2F src));
 12135   format %{ "FILD   $src\n\t"
 12136             "FSTP   $dst" %}
 12137   opcode(0xDB, 0x0);  /* DB /0 */
 12138   ins_encode( Push_Mem_I(src),
 12139               Pop_Reg_F(dst));
 12140   ins_pipe( fpu_reg_mem );
 12141 %}
 12143 // This instruction does not round to 24-bits
 12144 instruct convI2F_mem(regF dst, memory mem) %{
 12145   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
 12146   match(Set dst (ConvI2F (LoadI mem)));
 12147   format %{ "FILD   $mem\n\t"
 12148             "FSTP   $dst" %}
 12149   opcode(0xDB);      /* DB /0 */
 12150   ins_encode( OpcP, RMopc_Mem(0x00,mem),
 12151               Pop_Reg_F(dst));
 12152   ins_pipe( fpu_reg_mem );
 12153 %}
 12155 // Convert an int to a float in xmm; no rounding step needed.
 12156 instruct convI2X_reg(regX dst, eRegI src) %{
 12157   predicate( UseSSE==1 || UseSSE>=2 && !UseXmmI2F );
 12158   match(Set dst (ConvI2F src));
 12159   format %{ "CVTSI2SS $dst, $src" %}
 12161   opcode(0xF3, 0x0F, 0x2A);  /* F3 0F 2A /r */
 12162   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
 12163   ins_pipe( pipe_slow );
 12164 %}
 12166  instruct convXI2X_reg(regX dst, eRegI src)
 12167 %{
 12168   predicate( UseSSE>=2 && UseXmmI2F );
 12169   match(Set dst (ConvI2F src));
 12171   format %{ "MOVD  $dst,$src\n\t"
 12172             "CVTDQ2PS $dst,$dst\t# i2f" %}
 12173   ins_encode %{
 12174     __ movdl($dst$$XMMRegister, $src$$Register);
 12175     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
 12176   %}
 12177   ins_pipe(pipe_slow); // XXX
 12178 %}
 12180 instruct convI2L_reg( eRegL dst, eRegI src, eFlagsReg cr) %{
 12181   match(Set dst (ConvI2L src));
 12182   effect(KILL cr);
 12183   ins_cost(375);
 12184   format %{ "MOV    $dst.lo,$src\n\t"
 12185             "MOV    $dst.hi,$src\n\t"
 12186             "SAR    $dst.hi,31" %}
 12187   ins_encode(convert_int_long(dst,src));
 12188   ins_pipe( ialu_reg_reg_long );
 12189 %}
 12191 // Zero-extend convert int to long
 12192 instruct convI2L_reg_zex(eRegL dst, eRegI src, immL_32bits mask, eFlagsReg flags ) %{
 12193   match(Set dst (AndL (ConvI2L src) mask) );
 12194   effect( KILL flags );
 12195   ins_cost(250);
 12196   format %{ "MOV    $dst.lo,$src\n\t"
 12197             "XOR    $dst.hi,$dst.hi" %}
 12198   opcode(0x33); // XOR
 12199   ins_encode(enc_Copy(dst,src), OpcP, RegReg_Hi2(dst,dst) );
 12200   ins_pipe( ialu_reg_reg_long );
 12201 %}
 12203 // Zero-extend long
 12204 instruct zerox_long(eRegL dst, eRegL src, immL_32bits mask, eFlagsReg flags ) %{
 12205   match(Set dst (AndL src mask) );
 12206   effect( KILL flags );
 12207   ins_cost(250);
 12208   format %{ "MOV    $dst.lo,$src.lo\n\t"
 12209             "XOR    $dst.hi,$dst.hi\n\t" %}
 12210   opcode(0x33); // XOR
 12211   ins_encode(enc_Copy(dst,src), OpcP, RegReg_Hi2(dst,dst) );
 12212   ins_pipe( ialu_reg_reg_long );
 12213 %}
 12215 instruct convL2D_reg( stackSlotD dst, eRegL src, eFlagsReg cr) %{
 12216   predicate (UseSSE<=1);
 12217   match(Set dst (ConvL2D src));
 12218   effect( KILL cr );
 12219   format %{ "PUSH   $src.hi\t# Convert long to double\n\t"
 12220             "PUSH   $src.lo\n\t"
 12221             "FILD   ST,[ESP + #0]\n\t"
 12222             "ADD    ESP,8\n\t"
 12223             "FSTP_D $dst\t# D-round" %}
 12224   opcode(0xDF, 0x5);  /* DF /5 */
 12225   ins_encode(convert_long_double(src), Pop_Mem_D(dst));
 12226   ins_pipe( pipe_slow );
 12227 %}
 12229 instruct convL2XD_reg( regXD dst, eRegL src, eFlagsReg cr) %{
 12230   predicate (UseSSE>=2);
 12231   match(Set dst (ConvL2D src));
 12232   effect( KILL cr );
 12233   format %{ "PUSH   $src.hi\t# Convert long to double\n\t"
 12234             "PUSH   $src.lo\n\t"
 12235             "FILD_D [ESP]\n\t"
 12236             "FSTP_D [ESP]\n\t"
 12237             "MOVSD  $dst,[ESP]\n\t"
 12238             "ADD    ESP,8" %}
 12239   opcode(0xDF, 0x5);  /* DF /5 */
 12240   ins_encode(convert_long_double2(src), Push_ResultXD(dst));
 12241   ins_pipe( pipe_slow );
 12242 %}
 12244 instruct convL2X_reg( regX dst, eRegL src, eFlagsReg cr) %{
 12245   predicate (UseSSE>=1);
 12246   match(Set dst (ConvL2F src));
 12247   effect( KILL cr );
 12248   format %{ "PUSH   $src.hi\t# Convert long to single float\n\t"
 12249             "PUSH   $src.lo\n\t"
 12250             "FILD_D [ESP]\n\t"
 12251             "FSTP_S [ESP]\n\t"
 12252             "MOVSS  $dst,[ESP]\n\t"
 12253             "ADD    ESP,8" %}
 12254   opcode(0xDF, 0x5);  /* DF /5 */
 12255   ins_encode(convert_long_double2(src), Push_ResultX(dst,0x8));
 12256   ins_pipe( pipe_slow );
 12257 %}
 12259 instruct convL2F_reg( stackSlotF dst, eRegL src, eFlagsReg cr) %{
 12260   match(Set dst (ConvL2F src));
 12261   effect( KILL cr );
 12262   format %{ "PUSH   $src.hi\t# Convert long to single float\n\t"
 12263             "PUSH   $src.lo\n\t"
 12264             "FILD   ST,[ESP + #0]\n\t"
 12265             "ADD    ESP,8\n\t"
 12266             "FSTP_S $dst\t# F-round" %}
 12267   opcode(0xDF, 0x5);  /* DF /5 */
 12268   ins_encode(convert_long_double(src), Pop_Mem_F(dst));
 12269   ins_pipe( pipe_slow );
 12270 %}
 12272 instruct convL2I_reg( eRegI dst, eRegL src ) %{
 12273   match(Set dst (ConvL2I src));
 12274   effect( DEF dst, USE src );
 12275   format %{ "MOV    $dst,$src.lo" %}
 12276   ins_encode(enc_CopyL_Lo(dst,src));
 12277   ins_pipe( ialu_reg_reg );
 12278 %}
 12281 instruct MoveF2I_stack_reg(eRegI dst, stackSlotF src) %{
 12282   match(Set dst (MoveF2I src));
 12283   effect( DEF dst, USE src );
 12284   ins_cost(100);
 12285   format %{ "MOV    $dst,$src\t# MoveF2I_stack_reg" %}
 12286   opcode(0x8B);
 12287   ins_encode( OpcP, RegMem(dst,src));
 12288   ins_pipe( ialu_reg_mem );
 12289 %}
 12291 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
 12292   predicate(UseSSE==0);
 12293   match(Set dst (MoveF2I src));
 12294   effect( DEF dst, USE src );
 12296   ins_cost(125);
 12297   format %{ "FST_S  $dst,$src\t# MoveF2I_reg_stack" %}
 12298   ins_encode( Pop_Mem_Reg_F(dst, src) );
 12299   ins_pipe( fpu_mem_reg );
 12300 %}
 12302 instruct MoveF2I_reg_stack_sse(stackSlotI dst, regX src) %{
 12303   predicate(UseSSE>=1);
 12304   match(Set dst (MoveF2I src));
 12305   effect( DEF dst, USE src );
 12307   ins_cost(95);
 12308   format %{ "MOVSS  $dst,$src\t# MoveF2I_reg_stack_sse" %}
 12309   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x11), RegMem(src, dst));
 12310   ins_pipe( pipe_slow );
 12311 %}
 12313 instruct MoveF2I_reg_reg_sse(eRegI dst, regX src) %{
 12314   predicate(UseSSE>=2);
 12315   match(Set dst (MoveF2I src));
 12316   effect( DEF dst, USE src );
 12317   ins_cost(85);
 12318   format %{ "MOVD   $dst,$src\t# MoveF2I_reg_reg_sse" %}
 12319   ins_encode( MovX2I_reg(dst, src));
 12320   ins_pipe( pipe_slow );
 12321 %}
 12323 instruct MoveI2F_reg_stack(stackSlotF dst, eRegI src) %{
 12324   match(Set dst (MoveI2F src));
 12325   effect( DEF dst, USE src );
 12327   ins_cost(100);
 12328   format %{ "MOV    $dst,$src\t# MoveI2F_reg_stack" %}
 12329   opcode(0x89);
 12330   ins_encode( OpcPRegSS( dst, src ) );
 12331   ins_pipe( ialu_mem_reg );
 12332 %}
 12335 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
 12336   predicate(UseSSE==0);
 12337   match(Set dst (MoveI2F src));
 12338   effect(DEF dst, USE src);
 12340   ins_cost(125);
 12341   format %{ "FLD_S  $src\n\t"
 12342             "FSTP   $dst\t# MoveI2F_stack_reg" %}
 12343   opcode(0xD9);               /* D9 /0, FLD m32real */
 12344   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
 12345               Pop_Reg_F(dst) );
 12346   ins_pipe( fpu_reg_mem );
 12347 %}
 12349 instruct MoveI2F_stack_reg_sse(regX dst, stackSlotI src) %{
 12350   predicate(UseSSE>=1);
 12351   match(Set dst (MoveI2F src));
 12352   effect( DEF dst, USE src );
 12354   ins_cost(95);
 12355   format %{ "MOVSS  $dst,$src\t# MoveI2F_stack_reg_sse" %}
 12356   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x10), RegMem(dst,src));
 12357   ins_pipe( pipe_slow );
 12358 %}
 12360 instruct MoveI2F_reg_reg_sse(regX dst, eRegI src) %{
 12361   predicate(UseSSE>=2);
 12362   match(Set dst (MoveI2F src));
 12363   effect( DEF dst, USE src );
 12365   ins_cost(85);
 12366   format %{ "MOVD   $dst,$src\t# MoveI2F_reg_reg_sse" %}
 12367   ins_encode( MovI2X_reg(dst, src) );
 12368   ins_pipe( pipe_slow );
 12369 %}
 12371 instruct MoveD2L_stack_reg(eRegL dst, stackSlotD src) %{
 12372   match(Set dst (MoveD2L src));
 12373   effect(DEF dst, USE src);
 12375   ins_cost(250);
 12376   format %{ "MOV    $dst.lo,$src\n\t"
 12377             "MOV    $dst.hi,$src+4\t# MoveD2L_stack_reg" %}
 12378   opcode(0x8B, 0x8B);
 12379   ins_encode( OpcP, RegMem(dst,src), OpcS, RegMem_Hi(dst,src));
 12380   ins_pipe( ialu_mem_long_reg );
 12381 %}
 12383 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
 12384   predicate(UseSSE<=1);
 12385   match(Set dst (MoveD2L src));
 12386   effect(DEF dst, USE src);
 12388   ins_cost(125);
 12389   format %{ "FST_D  $dst,$src\t# MoveD2L_reg_stack" %}
 12390   ins_encode( Pop_Mem_Reg_D(dst, src) );
 12391   ins_pipe( fpu_mem_reg );
 12392 %}
 12394 instruct MoveD2L_reg_stack_sse(stackSlotL dst, regXD src) %{
 12395   predicate(UseSSE>=2);
 12396   match(Set dst (MoveD2L src));
 12397   effect(DEF dst, USE src);
 12398   ins_cost(95);
 12400   format %{ "MOVSD  $dst,$src\t# MoveD2L_reg_stack_sse" %}
 12401   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x11), RegMem(src,dst));
 12402   ins_pipe( pipe_slow );
 12403 %}
 12405 instruct MoveD2L_reg_reg_sse(eRegL dst, regXD src, regXD tmp) %{
 12406   predicate(UseSSE>=2);
 12407   match(Set dst (MoveD2L src));
 12408   effect(DEF dst, USE src, TEMP tmp);
 12409   ins_cost(85);
 12410   format %{ "MOVD   $dst.lo,$src\n\t"
 12411             "PSHUFLW $tmp,$src,0x4E\n\t"
 12412             "MOVD   $dst.hi,$tmp\t# MoveD2L_reg_reg_sse" %}
 12413   ins_encode( MovXD2L_reg(dst, src, tmp) );
 12414   ins_pipe( pipe_slow );
 12415 %}
 12417 instruct MoveL2D_reg_stack(stackSlotD dst, eRegL src) %{
 12418   match(Set dst (MoveL2D src));
 12419   effect(DEF dst, USE src);
 12421   ins_cost(200);
 12422   format %{ "MOV    $dst,$src.lo\n\t"
 12423             "MOV    $dst+4,$src.hi\t# MoveL2D_reg_stack" %}
 12424   opcode(0x89, 0x89);
 12425   ins_encode( OpcP, RegMem( src, dst ), OpcS, RegMem_Hi( src, dst ) );
 12426   ins_pipe( ialu_mem_long_reg );
 12427 %}
 12430 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
 12431   predicate(UseSSE<=1);
 12432   match(Set dst (MoveL2D src));
 12433   effect(DEF dst, USE src);
 12434   ins_cost(125);
 12436   format %{ "FLD_D  $src\n\t"
 12437             "FSTP   $dst\t# MoveL2D_stack_reg" %}
 12438   opcode(0xDD);               /* DD /0, FLD m64real */
 12439   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
 12440               Pop_Reg_D(dst) );
 12441   ins_pipe( fpu_reg_mem );
 12442 %}
 12445 instruct MoveL2D_stack_reg_sse(regXD dst, stackSlotL src) %{
 12446   predicate(UseSSE>=2 && UseXmmLoadAndClearUpper);
 12447   match(Set dst (MoveL2D src));
 12448   effect(DEF dst, USE src);
 12450   ins_cost(95);
 12451   format %{ "MOVSD  $dst,$src\t# MoveL2D_stack_reg_sse" %}
 12452   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x10), RegMem(dst,src));
 12453   ins_pipe( pipe_slow );
 12454 %}
 12456 instruct MoveL2D_stack_reg_sse_partial(regXD dst, stackSlotL src) %{
 12457   predicate(UseSSE>=2 && !UseXmmLoadAndClearUpper);
 12458   match(Set dst (MoveL2D src));
 12459   effect(DEF dst, USE src);
 12461   ins_cost(95);
 12462   format %{ "MOVLPD $dst,$src\t# MoveL2D_stack_reg_sse" %}
 12463   ins_encode( Opcode(0x66), Opcode(0x0F), Opcode(0x12), RegMem(dst,src));
 12464   ins_pipe( pipe_slow );
 12465 %}
 12467 instruct MoveL2D_reg_reg_sse(regXD dst, eRegL src, regXD tmp) %{
 12468   predicate(UseSSE>=2);
 12469   match(Set dst (MoveL2D src));
 12470   effect(TEMP dst, USE src, TEMP tmp);
 12471   ins_cost(85);
 12472   format %{ "MOVD   $dst,$src.lo\n\t"
 12473             "MOVD   $tmp,$src.hi\n\t"
 12474             "PUNPCKLDQ $dst,$tmp\t# MoveL2D_reg_reg_sse" %}
 12475   ins_encode( MovL2XD_reg(dst, src, tmp) );
 12476   ins_pipe( pipe_slow );
 12477 %}
 12479 // Replicate scalar to packed byte (1 byte) values in xmm
 12480 instruct Repl8B_reg(regXD dst, regXD src) %{
 12481   predicate(UseSSE>=2);
 12482   match(Set dst (Replicate8B src));
 12483   format %{ "MOVDQA  $dst,$src\n\t"
 12484             "PUNPCKLBW $dst,$dst\n\t"
 12485             "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
 12486   ins_encode( pshufd_8x8(dst, src));
 12487   ins_pipe( pipe_slow );
 12488 %}
 12490 // Replicate scalar to packed byte (1 byte) values in xmm
 12491 instruct Repl8B_eRegI(regXD dst, eRegI src) %{
 12492   predicate(UseSSE>=2);
 12493   match(Set dst (Replicate8B src));
 12494   format %{ "MOVD    $dst,$src\n\t"
 12495             "PUNPCKLBW $dst,$dst\n\t"
 12496             "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
 12497   ins_encode( mov_i2x(dst, src), pshufd_8x8(dst, dst));
 12498   ins_pipe( pipe_slow );
 12499 %}
 12501 // Replicate scalar zero to packed byte (1 byte) values in xmm
 12502 instruct Repl8B_immI0(regXD dst, immI0 zero) %{
 12503   predicate(UseSSE>=2);
 12504   match(Set dst (Replicate8B zero));
 12505   format %{ "PXOR  $dst,$dst\t! replicate8B" %}
 12506   ins_encode( pxor(dst, dst));
 12507   ins_pipe( fpu_reg_reg );
 12508 %}
 12510 // Replicate scalar to packed shore (2 byte) values in xmm
 12511 instruct Repl4S_reg(regXD dst, regXD src) %{
 12512   predicate(UseSSE>=2);
 12513   match(Set dst (Replicate4S src));
 12514   format %{ "PSHUFLW $dst,$src,0x00\t! replicate4S" %}
 12515   ins_encode( pshufd_4x16(dst, src));
 12516   ins_pipe( fpu_reg_reg );
 12517 %}
 12519 // Replicate scalar to packed shore (2 byte) values in xmm
 12520 instruct Repl4S_eRegI(regXD dst, eRegI src) %{
 12521   predicate(UseSSE>=2);
 12522   match(Set dst (Replicate4S src));
 12523   format %{ "MOVD    $dst,$src\n\t"
 12524             "PSHUFLW $dst,$dst,0x00\t! replicate4S" %}
 12525   ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
 12526   ins_pipe( fpu_reg_reg );
 12527 %}
 12529 // Replicate scalar zero to packed short (2 byte) values in xmm
 12530 instruct Repl4S_immI0(regXD dst, immI0 zero) %{
 12531   predicate(UseSSE>=2);
 12532   match(Set dst (Replicate4S zero));
 12533   format %{ "PXOR  $dst,$dst\t! replicate4S" %}
 12534   ins_encode( pxor(dst, dst));
 12535   ins_pipe( fpu_reg_reg );
 12536 %}
 12538 // Replicate scalar to packed char (2 byte) values in xmm
 12539 instruct Repl4C_reg(regXD dst, regXD src) %{
 12540   predicate(UseSSE>=2);
 12541   match(Set dst (Replicate4C src));
 12542   format %{ "PSHUFLW $dst,$src,0x00\t! replicate4C" %}
 12543   ins_encode( pshufd_4x16(dst, src));
 12544   ins_pipe( fpu_reg_reg );
 12545 %}
 12547 // Replicate scalar to packed char (2 byte) values in xmm
 12548 instruct Repl4C_eRegI(regXD dst, eRegI src) %{
 12549   predicate(UseSSE>=2);
 12550   match(Set dst (Replicate4C src));
 12551   format %{ "MOVD    $dst,$src\n\t"
 12552             "PSHUFLW $dst,$dst,0x00\t! replicate4C" %}
 12553   ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
 12554   ins_pipe( fpu_reg_reg );
 12555 %}
 12557 // Replicate scalar zero to packed char (2 byte) values in xmm
 12558 instruct Repl4C_immI0(regXD dst, immI0 zero) %{
 12559   predicate(UseSSE>=2);
 12560   match(Set dst (Replicate4C zero));
 12561   format %{ "PXOR  $dst,$dst\t! replicate4C" %}
 12562   ins_encode( pxor(dst, dst));
 12563   ins_pipe( fpu_reg_reg );
 12564 %}
 12566 // Replicate scalar to packed integer (4 byte) values in xmm
 12567 instruct Repl2I_reg(regXD dst, regXD src) %{
 12568   predicate(UseSSE>=2);
 12569   match(Set dst (Replicate2I src));
 12570   format %{ "PSHUFD $dst,$src,0x00\t! replicate2I" %}
 12571   ins_encode( pshufd(dst, src, 0x00));
 12572   ins_pipe( fpu_reg_reg );
 12573 %}
 12575 // Replicate scalar to packed integer (4 byte) values in xmm
 12576 instruct Repl2I_eRegI(regXD dst, eRegI src) %{
 12577   predicate(UseSSE>=2);
 12578   match(Set dst (Replicate2I src));
 12579   format %{ "MOVD   $dst,$src\n\t"
 12580             "PSHUFD $dst,$dst,0x00\t! replicate2I" %}
 12581   ins_encode( mov_i2x(dst, src), pshufd(dst, dst, 0x00));
 12582   ins_pipe( fpu_reg_reg );
 12583 %}
 12585 // Replicate scalar zero to packed integer (2 byte) values in xmm
 12586 instruct Repl2I_immI0(regXD dst, immI0 zero) %{
 12587   predicate(UseSSE>=2);
 12588   match(Set dst (Replicate2I zero));
 12589   format %{ "PXOR  $dst,$dst\t! replicate2I" %}
 12590   ins_encode( pxor(dst, dst));
 12591   ins_pipe( fpu_reg_reg );
 12592 %}
 12594 // Replicate scalar to packed single precision floating point values in xmm
 12595 instruct Repl2F_reg(regXD dst, regXD src) %{
 12596   predicate(UseSSE>=2);
 12597   match(Set dst (Replicate2F src));
 12598   format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
 12599   ins_encode( pshufd(dst, src, 0xe0));
 12600   ins_pipe( fpu_reg_reg );
 12601 %}
 12603 // Replicate scalar to packed single precision floating point values in xmm
 12604 instruct Repl2F_regX(regXD dst, regX src) %{
 12605   predicate(UseSSE>=2);
 12606   match(Set dst (Replicate2F src));
 12607   format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
 12608   ins_encode( pshufd(dst, src, 0xe0));
 12609   ins_pipe( fpu_reg_reg );
 12610 %}
 12612 // Replicate scalar to packed single precision floating point values in xmm
 12613 instruct Repl2F_immXF0(regXD dst, immXF0 zero) %{
 12614   predicate(UseSSE>=2);
 12615   match(Set dst (Replicate2F zero));
 12616   format %{ "PXOR  $dst,$dst\t! replicate2F" %}
 12617   ins_encode( pxor(dst, dst));
 12618   ins_pipe( fpu_reg_reg );
 12619 %}
 12621 // =======================================================================
 12622 // fast clearing of an array
 12623 instruct rep_stos(eCXRegI cnt, eDIRegP base, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
 12624   match(Set dummy (ClearArray cnt base));
 12625   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
 12626   format %{ "SHL    ECX,1\t# Convert doublewords to words\n\t"
 12627             "XOR    EAX,EAX\n\t"
 12628             "REP STOS\t# store EAX into [EDI++] while ECX--" %}
 12629   opcode(0,0x4);
 12630   ins_encode( Opcode(0xD1), RegOpc(ECX),
 12631               OpcRegReg(0x33,EAX,EAX),
 12632               Opcode(0xF3), Opcode(0xAB) );
 12633   ins_pipe( pipe_slow );
 12634 %}
 12636 instruct string_compare(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eDXRegI cnt2,
 12637                         eAXRegI result, regXD tmp1, eFlagsReg cr) %{
 12638   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
 12639   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
 12641   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
 12642   ins_encode %{
 12643     __ string_compare($str1$$Register, $str2$$Register,
 12644                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
 12645                       $tmp1$$XMMRegister);
 12646   %}
 12647   ins_pipe( pipe_slow );
 12648 %}
 12650 // fast string equals
 12651 instruct string_equals(eDIRegP str1, eSIRegP str2, eCXRegI cnt, eAXRegI result,
 12652                        regXD tmp1, regXD tmp2, eBXRegI tmp3, eFlagsReg cr) %{
 12653   match(Set result (StrEquals (Binary str1 str2) cnt));
 12654   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
 12656   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
 12657   ins_encode %{
 12658     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
 12659                           $cnt$$Register, $result$$Register, $tmp3$$Register,
 12660                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
 12661   %}
 12662   ins_pipe( pipe_slow );
 12663 %}
 12665 // fast search of substring with known size.
 12666 instruct string_indexof_con(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, immI int_cnt2,
 12667                             eBXRegI result, regXD vec, eAXRegI cnt2, eCXRegI tmp, eFlagsReg cr) %{
 12668   predicate(UseSSE42Intrinsics);
 12669   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
 12670   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
 12672   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
 12673   ins_encode %{
 12674     int icnt2 = (int)$int_cnt2$$constant;
 12675     if (icnt2 >= 8) {
 12676       // IndexOf for constant substrings with size >= 8 elements
 12677       // which don't need to be loaded through stack.
 12678       __ string_indexofC8($str1$$Register, $str2$$Register,
 12679                           $cnt1$$Register, $cnt2$$Register,
 12680                           icnt2, $result$$Register,
 12681                           $vec$$XMMRegister, $tmp$$Register);
 12682     } else {
 12683       // Small strings are loaded through stack if they cross page boundary.
 12684       __ string_indexof($str1$$Register, $str2$$Register,
 12685                         $cnt1$$Register, $cnt2$$Register,
 12686                         icnt2, $result$$Register,
 12687                         $vec$$XMMRegister, $tmp$$Register);
 12689   %}
 12690   ins_pipe( pipe_slow );
 12691 %}
 12693 instruct string_indexof(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, eAXRegI cnt2,
 12694                         eBXRegI result, regXD vec, eCXRegI tmp, eFlagsReg cr) %{
 12695   predicate(UseSSE42Intrinsics);
 12696   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
 12697   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
 12699   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
 12700   ins_encode %{
 12701     __ string_indexof($str1$$Register, $str2$$Register,
 12702                       $cnt1$$Register, $cnt2$$Register,
 12703                       (-1), $result$$Register,
 12704                       $vec$$XMMRegister, $tmp$$Register);
 12705   %}
 12706   ins_pipe( pipe_slow );
 12707 %}
 12709 // fast array equals
 12710 instruct array_equals(eDIRegP ary1, eSIRegP ary2, eAXRegI result,
 12711                       regXD tmp1, regXD tmp2, eCXRegI tmp3, eBXRegI tmp4, eFlagsReg cr)
 12712 %{
 12713   match(Set result (AryEq ary1 ary2));
 12714   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
 12715   //ins_cost(300);
 12717   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
 12718   ins_encode %{
 12719     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
 12720                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
 12721                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
 12722   %}
 12723   ins_pipe( pipe_slow );
 12724 %}
 12726 //----------Control Flow Instructions------------------------------------------
 12727 // Signed compare Instructions
 12728 instruct compI_eReg(eFlagsReg cr, eRegI op1, eRegI op2) %{
 12729   match(Set cr (CmpI op1 op2));
 12730   effect( DEF cr, USE op1, USE op2 );
 12731   format %{ "CMP    $op1,$op2" %}
 12732   opcode(0x3B);  /* Opcode 3B /r */
 12733   ins_encode( OpcP, RegReg( op1, op2) );
 12734   ins_pipe( ialu_cr_reg_reg );
 12735 %}
 12737 instruct compI_eReg_imm(eFlagsReg cr, eRegI op1, immI op2) %{
 12738   match(Set cr (CmpI op1 op2));
 12739   effect( DEF cr, USE op1 );
 12740   format %{ "CMP    $op1,$op2" %}
 12741   opcode(0x81,0x07);  /* Opcode 81 /7 */
 12742   // ins_encode( RegImm( op1, op2) );  /* Was CmpImm */
 12743   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
 12744   ins_pipe( ialu_cr_reg_imm );
 12745 %}
 12747 // Cisc-spilled version of cmpI_eReg
 12748 instruct compI_eReg_mem(eFlagsReg cr, eRegI op1, memory op2) %{
 12749   match(Set cr (CmpI op1 (LoadI op2)));
 12751   format %{ "CMP    $op1,$op2" %}
 12752   ins_cost(500);
 12753   opcode(0x3B);  /* Opcode 3B /r */
 12754   ins_encode( OpcP, RegMem( op1, op2) );
 12755   ins_pipe( ialu_cr_reg_mem );
 12756 %}
 12758 instruct testI_reg( eFlagsReg cr, eRegI src, immI0 zero ) %{
 12759   match(Set cr (CmpI src zero));
 12760   effect( DEF cr, USE src );
 12762   format %{ "TEST   $src,$src" %}
 12763   opcode(0x85);
 12764   ins_encode( OpcP, RegReg( src, src ) );
 12765   ins_pipe( ialu_cr_reg_imm );
 12766 %}
 12768 instruct testI_reg_imm( eFlagsReg cr, eRegI src, immI con, immI0 zero ) %{
 12769   match(Set cr (CmpI (AndI src con) zero));
 12771   format %{ "TEST   $src,$con" %}
 12772   opcode(0xF7,0x00);
 12773   ins_encode( OpcP, RegOpc(src), Con32(con) );
 12774   ins_pipe( ialu_cr_reg_imm );
 12775 %}
 12777 instruct testI_reg_mem( eFlagsReg cr, eRegI src, memory mem, immI0 zero ) %{
 12778   match(Set cr (CmpI (AndI src mem) zero));
 12780   format %{ "TEST   $src,$mem" %}
 12781   opcode(0x85);
 12782   ins_encode( OpcP, RegMem( src, mem ) );
 12783   ins_pipe( ialu_cr_reg_mem );
 12784 %}
 12786 // Unsigned compare Instructions; really, same as signed except they
 12787 // produce an eFlagsRegU instead of eFlagsReg.
 12788 instruct compU_eReg(eFlagsRegU cr, eRegI op1, eRegI op2) %{
 12789   match(Set cr (CmpU op1 op2));
 12791   format %{ "CMPu   $op1,$op2" %}
 12792   opcode(0x3B);  /* Opcode 3B /r */
 12793   ins_encode( OpcP, RegReg( op1, op2) );
 12794   ins_pipe( ialu_cr_reg_reg );
 12795 %}
 12797 instruct compU_eReg_imm(eFlagsRegU cr, eRegI op1, immI op2) %{
 12798   match(Set cr (CmpU op1 op2));
 12800   format %{ "CMPu   $op1,$op2" %}
 12801   opcode(0x81,0x07);  /* Opcode 81 /7 */
 12802   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
 12803   ins_pipe( ialu_cr_reg_imm );
 12804 %}
 12806 // // Cisc-spilled version of cmpU_eReg
 12807 instruct compU_eReg_mem(eFlagsRegU cr, eRegI op1, memory op2) %{
 12808   match(Set cr (CmpU op1 (LoadI op2)));
 12810   format %{ "CMPu   $op1,$op2" %}
 12811   ins_cost(500);
 12812   opcode(0x3B);  /* Opcode 3B /r */
 12813   ins_encode( OpcP, RegMem( op1, op2) );
 12814   ins_pipe( ialu_cr_reg_mem );
 12815 %}
 12817 // // Cisc-spilled version of cmpU_eReg
 12818 //instruct compU_mem_eReg(eFlagsRegU cr, memory op1, eRegI op2) %{
 12819 //  match(Set cr (CmpU (LoadI op1) op2));
 12820 //
 12821 //  format %{ "CMPu   $op1,$op2" %}
 12822 //  ins_cost(500);
 12823 //  opcode(0x39);  /* Opcode 39 /r */
 12824 //  ins_encode( OpcP, RegMem( op1, op2) );
 12825 //%}
 12827 instruct testU_reg( eFlagsRegU cr, eRegI src, immI0 zero ) %{
 12828   match(Set cr (CmpU src zero));
 12830   format %{ "TESTu  $src,$src" %}
 12831   opcode(0x85);
 12832   ins_encode( OpcP, RegReg( src, src ) );
 12833   ins_pipe( ialu_cr_reg_imm );
 12834 %}
 12836 // Unsigned pointer compare Instructions
 12837 instruct compP_eReg(eFlagsRegU cr, eRegP op1, eRegP op2) %{
 12838   match(Set cr (CmpP op1 op2));
 12840   format %{ "CMPu   $op1,$op2" %}
 12841   opcode(0x3B);  /* Opcode 3B /r */
 12842   ins_encode( OpcP, RegReg( op1, op2) );
 12843   ins_pipe( ialu_cr_reg_reg );
 12844 %}
 12846 instruct compP_eReg_imm(eFlagsRegU cr, eRegP op1, immP op2) %{
 12847   match(Set cr (CmpP op1 op2));
 12849   format %{ "CMPu   $op1,$op2" %}
 12850   opcode(0x81,0x07);  /* Opcode 81 /7 */
 12851   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
 12852   ins_pipe( ialu_cr_reg_imm );
 12853 %}
 12855 // // Cisc-spilled version of cmpP_eReg
 12856 instruct compP_eReg_mem(eFlagsRegU cr, eRegP op1, memory op2) %{
 12857   match(Set cr (CmpP op1 (LoadP op2)));
 12859   format %{ "CMPu   $op1,$op2" %}
 12860   ins_cost(500);
 12861   opcode(0x3B);  /* Opcode 3B /r */
 12862   ins_encode( OpcP, RegMem( op1, op2) );
 12863   ins_pipe( ialu_cr_reg_mem );
 12864 %}
 12866 // // Cisc-spilled version of cmpP_eReg
 12867 //instruct compP_mem_eReg(eFlagsRegU cr, memory op1, eRegP op2) %{
 12868 //  match(Set cr (CmpP (LoadP op1) op2));
 12869 //
 12870 //  format %{ "CMPu   $op1,$op2" %}
 12871 //  ins_cost(500);
 12872 //  opcode(0x39);  /* Opcode 39 /r */
 12873 //  ins_encode( OpcP, RegMem( op1, op2) );
 12874 //%}
 12876 // Compare raw pointer (used in out-of-heap check).
 12877 // Only works because non-oop pointers must be raw pointers
 12878 // and raw pointers have no anti-dependencies.
 12879 instruct compP_mem_eReg( eFlagsRegU cr, eRegP op1, memory op2 ) %{
 12880   predicate( !n->in(2)->in(2)->bottom_type()->isa_oop_ptr() );
 12881   match(Set cr (CmpP op1 (LoadP op2)));
 12883   format %{ "CMPu   $op1,$op2" %}
 12884   opcode(0x3B);  /* Opcode 3B /r */
 12885   ins_encode( OpcP, RegMem( op1, op2) );
 12886   ins_pipe( ialu_cr_reg_mem );
 12887 %}
 12889 //
 12890 // This will generate a signed flags result. This should be ok
 12891 // since any compare to a zero should be eq/neq.
 12892 instruct testP_reg( eFlagsReg cr, eRegP src, immP0 zero ) %{
 12893   match(Set cr (CmpP src zero));
 12895   format %{ "TEST   $src,$src" %}
 12896   opcode(0x85);
 12897   ins_encode( OpcP, RegReg( src, src ) );
 12898   ins_pipe( ialu_cr_reg_imm );
 12899 %}
 12901 // Cisc-spilled version of testP_reg
 12902 // This will generate a signed flags result. This should be ok
 12903 // since any compare to a zero should be eq/neq.
 12904 instruct testP_Reg_mem( eFlagsReg cr, memory op, immI0 zero ) %{
 12905   match(Set cr (CmpP (LoadP op) zero));
 12907   format %{ "TEST   $op,0xFFFFFFFF" %}
 12908   ins_cost(500);
 12909   opcode(0xF7);               /* Opcode F7 /0 */
 12910   ins_encode( OpcP, RMopc_Mem(0x00,op), Con_d32(0xFFFFFFFF) );
 12911   ins_pipe( ialu_cr_reg_imm );
 12912 %}
 12914 // Yanked all unsigned pointer compare operations.
 12915 // Pointer compares are done with CmpP which is already unsigned.
 12917 //----------Max and Min--------------------------------------------------------
 12918 // Min Instructions
 12919 ////
 12920 //   *** Min and Max using the conditional move are slower than the
 12921 //   *** branch version on a Pentium III.
 12922 // // Conditional move for min
 12923 //instruct cmovI_reg_lt( eRegI op2, eRegI op1, eFlagsReg cr ) %{
 12924 //  effect( USE_DEF op2, USE op1, USE cr );
 12925 //  format %{ "CMOVlt $op2,$op1\t! min" %}
 12926 //  opcode(0x4C,0x0F);
 12927 //  ins_encode( OpcS, OpcP, RegReg( op2, op1 ) );
 12928 //  ins_pipe( pipe_cmov_reg );
 12929 //%}
 12930 //
 12931 //// Min Register with Register (P6 version)
 12932 //instruct minI_eReg_p6( eRegI op1, eRegI op2 ) %{
 12933 //  predicate(VM_Version::supports_cmov() );
 12934 //  match(Set op2 (MinI op1 op2));
 12935 //  ins_cost(200);
 12936 //  expand %{
 12937 //    eFlagsReg cr;
 12938 //    compI_eReg(cr,op1,op2);
 12939 //    cmovI_reg_lt(op2,op1,cr);
 12940 //  %}
 12941 //%}
 12943 // Min Register with Register (generic version)
 12944 instruct minI_eReg(eRegI dst, eRegI src, eFlagsReg flags) %{
 12945   match(Set dst (MinI dst src));
 12946   effect(KILL flags);
 12947   ins_cost(300);
 12949   format %{ "MIN    $dst,$src" %}
 12950   opcode(0xCC);
 12951   ins_encode( min_enc(dst,src) );
 12952   ins_pipe( pipe_slow );
 12953 %}
 12955 // Max Register with Register
 12956 //   *** Min and Max using the conditional move are slower than the
 12957 //   *** branch version on a Pentium III.
 12958 // // Conditional move for max
 12959 //instruct cmovI_reg_gt( eRegI op2, eRegI op1, eFlagsReg cr ) %{
 12960 //  effect( USE_DEF op2, USE op1, USE cr );
 12961 //  format %{ "CMOVgt $op2,$op1\t! max" %}
 12962 //  opcode(0x4F,0x0F);
 12963 //  ins_encode( OpcS, OpcP, RegReg( op2, op1 ) );
 12964 //  ins_pipe( pipe_cmov_reg );
 12965 //%}
 12966 //
 12967 // // Max Register with Register (P6 version)
 12968 //instruct maxI_eReg_p6( eRegI op1, eRegI op2 ) %{
 12969 //  predicate(VM_Version::supports_cmov() );
 12970 //  match(Set op2 (MaxI op1 op2));
 12971 //  ins_cost(200);
 12972 //  expand %{
 12973 //    eFlagsReg cr;
 12974 //    compI_eReg(cr,op1,op2);
 12975 //    cmovI_reg_gt(op2,op1,cr);
 12976 //  %}
 12977 //%}
 12979 // Max Register with Register (generic version)
 12980 instruct maxI_eReg(eRegI dst, eRegI src, eFlagsReg flags) %{
 12981   match(Set dst (MaxI dst src));
 12982   effect(KILL flags);
 12983   ins_cost(300);
 12985   format %{ "MAX    $dst,$src" %}
 12986   opcode(0xCC);
 12987   ins_encode( max_enc(dst,src) );
 12988   ins_pipe( pipe_slow );
 12989 %}
 12991 // ============================================================================
 12992 // Branch Instructions
 12993 // Jump Table
 12994 instruct jumpXtnd(eRegI switch_val) %{
 12995   match(Jump switch_val);
 12996   ins_cost(350);
 12997   format %{  "JMP    [$constantaddress](,$switch_val,1)\n\t" %}
 12998   ins_encode %{
 12999     // Jump to Address(table_base + switch_reg)
 13000     Address index(noreg, $switch_val$$Register, Address::times_1);
 13001     __ jump(ArrayAddress($constantaddress, index));
 13002   %}
 13003   ins_pc_relative(1);
 13004   ins_pipe(pipe_jmp);
 13005 %}
 13007 // Jump Direct - Label defines a relative address from JMP+1
 13008 instruct jmpDir(label labl) %{
 13009   match(Goto);
 13010   effect(USE labl);
 13012   ins_cost(300);
 13013   format %{ "JMP    $labl" %}
 13014   size(5);
 13015   opcode(0xE9);
 13016   ins_encode( OpcP, Lbl( labl ) );
 13017   ins_pipe( pipe_jmp );
 13018   ins_pc_relative(1);
 13019 %}
 13021 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 13022 instruct jmpCon(cmpOp cop, eFlagsReg cr, label labl) %{
 13023   match(If cop cr);
 13024   effect(USE labl);
 13026   ins_cost(300);
 13027   format %{ "J$cop    $labl" %}
 13028   size(6);
 13029   opcode(0x0F, 0x80);
 13030   ins_encode( Jcc( cop, labl) );
 13031   ins_pipe( pipe_jcc );
 13032   ins_pc_relative(1);
 13033 %}
 13035 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 13036 instruct jmpLoopEnd(cmpOp cop, eFlagsReg cr, label labl) %{
 13037   match(CountedLoopEnd cop cr);
 13038   effect(USE labl);
 13040   ins_cost(300);
 13041   format %{ "J$cop    $labl\t# Loop end" %}
 13042   size(6);
 13043   opcode(0x0F, 0x80);
 13044   ins_encode( Jcc( cop, labl) );
 13045   ins_pipe( pipe_jcc );
 13046   ins_pc_relative(1);
 13047 %}
 13049 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 13050 instruct jmpLoopEndU(cmpOpU cop, eFlagsRegU cmp, label labl) %{
 13051   match(CountedLoopEnd cop cmp);
 13052   effect(USE labl);
 13054   ins_cost(300);
 13055   format %{ "J$cop,u  $labl\t# Loop end" %}
 13056   size(6);
 13057   opcode(0x0F, 0x80);
 13058   ins_encode( Jcc( cop, labl) );
 13059   ins_pipe( pipe_jcc );
 13060   ins_pc_relative(1);
 13061 %}
 13063 instruct jmpLoopEndUCF(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
 13064   match(CountedLoopEnd cop cmp);
 13065   effect(USE labl);
 13067   ins_cost(200);
 13068   format %{ "J$cop,u  $labl\t# Loop end" %}
 13069   size(6);
 13070   opcode(0x0F, 0x80);
 13071   ins_encode( Jcc( cop, labl) );
 13072   ins_pipe( pipe_jcc );
 13073   ins_pc_relative(1);
 13074 %}
 13076 // Jump Direct Conditional - using unsigned comparison
 13077 instruct jmpConU(cmpOpU cop, eFlagsRegU cmp, label labl) %{
 13078   match(If cop cmp);
 13079   effect(USE labl);
 13081   ins_cost(300);
 13082   format %{ "J$cop,u  $labl" %}
 13083   size(6);
 13084   opcode(0x0F, 0x80);
 13085   ins_encode(Jcc(cop, labl));
 13086   ins_pipe(pipe_jcc);
 13087   ins_pc_relative(1);
 13088 %}
 13090 instruct jmpConUCF(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
 13091   match(If cop cmp);
 13092   effect(USE labl);
 13094   ins_cost(200);
 13095   format %{ "J$cop,u  $labl" %}
 13096   size(6);
 13097   opcode(0x0F, 0x80);
 13098   ins_encode(Jcc(cop, labl));
 13099   ins_pipe(pipe_jcc);
 13100   ins_pc_relative(1);
 13101 %}
 13103 instruct jmpConUCF2(cmpOpUCF2 cop, eFlagsRegUCF cmp, label labl) %{
 13104   match(If cop cmp);
 13105   effect(USE labl);
 13107   ins_cost(200);
 13108   format %{ $$template
 13109     if ($cop$$cmpcode == Assembler::notEqual) {
 13110       $$emit$$"JP,u   $labl\n\t"
 13111       $$emit$$"J$cop,u   $labl"
 13112     } else {
 13113       $$emit$$"JP,u   done\n\t"
 13114       $$emit$$"J$cop,u   $labl\n\t"
 13115       $$emit$$"done:"
 13117   %}
 13118   size(12);
 13119   opcode(0x0F, 0x80);
 13120   ins_encode %{
 13121     Label* l = $labl$$label;
 13122     $$$emit8$primary;
 13123     emit_cc(cbuf, $secondary, Assembler::parity);
 13124     int parity_disp = -1;
 13125     bool ok = false;
 13126     if ($cop$$cmpcode == Assembler::notEqual) {
 13127        // the two jumps 6 bytes apart so the jump distances are too
 13128        parity_disp = l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0;
 13129     } else if ($cop$$cmpcode == Assembler::equal) {
 13130        parity_disp = 6;
 13131        ok = true;
 13132     } else {
 13133        ShouldNotReachHere();
 13135     emit_d32(cbuf, parity_disp);
 13136     $$$emit8$primary;
 13137     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 13138     int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 4)) : 0;
 13139     emit_d32(cbuf, disp);
 13140   %}
 13141   ins_pipe(pipe_jcc);
 13142   ins_pc_relative(1);
 13143 %}
 13145 // ============================================================================
 13146 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary superklass
 13147 // array for an instance of the superklass.  Set a hidden internal cache on a
 13148 // hit (cache is checked with exposed code in gen_subtype_check()).  Return
 13149 // NZ for a miss or zero for a hit.  The encoding ALSO sets flags.
 13150 instruct partialSubtypeCheck( eDIRegP result, eSIRegP sub, eAXRegP super, eCXRegI rcx, eFlagsReg cr ) %{
 13151   match(Set result (PartialSubtypeCheck sub super));
 13152   effect( KILL rcx, KILL cr );
 13154   ins_cost(1100);  // slightly larger than the next version
 13155   format %{ "MOV    EDI,[$sub+Klass::secondary_supers]\n\t"
 13156             "MOV    ECX,[EDI+arrayKlass::length]\t# length to scan\n\t"
 13157             "ADD    EDI,arrayKlass::base_offset\t# Skip to start of data; set NZ in case count is zero\n\t"
 13158             "REPNE SCASD\t# Scan *EDI++ for a match with EAX while CX-- != 0\n\t"
 13159             "JNE,s  miss\t\t# Missed: EDI not-zero\n\t"
 13160             "MOV    [$sub+Klass::secondary_super_cache],$super\t# Hit: update cache\n\t"
 13161             "XOR    $result,$result\t\t Hit: EDI zero\n\t"
 13162      "miss:\t" %}
 13164   opcode(0x1); // Force a XOR of EDI
 13165   ins_encode( enc_PartialSubtypeCheck() );
 13166   ins_pipe( pipe_slow );
 13167 %}
 13169 instruct partialSubtypeCheck_vs_Zero( eFlagsReg cr, eSIRegP sub, eAXRegP super, eCXRegI rcx, eDIRegP result, immP0 zero ) %{
 13170   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
 13171   effect( KILL rcx, KILL result );
 13173   ins_cost(1000);
 13174   format %{ "MOV    EDI,[$sub+Klass::secondary_supers]\n\t"
 13175             "MOV    ECX,[EDI+arrayKlass::length]\t# length to scan\n\t"
 13176             "ADD    EDI,arrayKlass::base_offset\t# Skip to start of data; set NZ in case count is zero\n\t"
 13177             "REPNE SCASD\t# Scan *EDI++ for a match with EAX while CX-- != 0\n\t"
 13178             "JNE,s  miss\t\t# Missed: flags NZ\n\t"
 13179             "MOV    [$sub+Klass::secondary_super_cache],$super\t# Hit: update cache, flags Z\n\t"
 13180      "miss:\t" %}
 13182   opcode(0x0);  // No need to XOR EDI
 13183   ins_encode( enc_PartialSubtypeCheck() );
 13184   ins_pipe( pipe_slow );
 13185 %}
 13187 // ============================================================================
 13188 // Branch Instructions -- short offset versions
 13189 //
 13190 // These instructions are used to replace jumps of a long offset (the default
 13191 // match) with jumps of a shorter offset.  These instructions are all tagged
 13192 // with the ins_short_branch attribute, which causes the ADLC to suppress the
 13193 // match rules in general matching.  Instead, the ADLC generates a conversion
 13194 // method in the MachNode which can be used to do in-place replacement of the
 13195 // long variant with the shorter variant.  The compiler will determine if a
 13196 // branch can be taken by the is_short_branch_offset() predicate in the machine
 13197 // specific code section of the file.
 13199 // Jump Direct - Label defines a relative address from JMP+1
 13200 instruct jmpDir_short(label labl) %{
 13201   match(Goto);
 13202   effect(USE labl);
 13204   ins_cost(300);
 13205   format %{ "JMP,s  $labl" %}
 13206   size(2);
 13207   opcode(0xEB);
 13208   ins_encode( OpcP, LblShort( labl ) );
 13209   ins_pipe( pipe_jmp );
 13210   ins_pc_relative(1);
 13211   ins_short_branch(1);
 13212 %}
 13214 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 13215 instruct jmpCon_short(cmpOp cop, eFlagsReg cr, label labl) %{
 13216   match(If cop cr);
 13217   effect(USE labl);
 13219   ins_cost(300);
 13220   format %{ "J$cop,s  $labl" %}
 13221   size(2);
 13222   opcode(0x70);
 13223   ins_encode( JccShort( cop, labl) );
 13224   ins_pipe( pipe_jcc );
 13225   ins_pc_relative(1);
 13226   ins_short_branch(1);
 13227 %}
 13229 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 13230 instruct jmpLoopEnd_short(cmpOp cop, eFlagsReg cr, label labl) %{
 13231   match(CountedLoopEnd cop cr);
 13232   effect(USE labl);
 13234   ins_cost(300);
 13235   format %{ "J$cop,s  $labl\t# Loop end" %}
 13236   size(2);
 13237   opcode(0x70);
 13238   ins_encode( JccShort( cop, labl) );
 13239   ins_pipe( pipe_jcc );
 13240   ins_pc_relative(1);
 13241   ins_short_branch(1);
 13242 %}
 13244 // Jump Direct Conditional - Label defines a relative address from Jcc+1
 13245 instruct jmpLoopEndU_short(cmpOpU cop, eFlagsRegU cmp, label labl) %{
 13246   match(CountedLoopEnd cop cmp);
 13247   effect(USE labl);
 13249   ins_cost(300);
 13250   format %{ "J$cop,us $labl\t# Loop end" %}
 13251   size(2);
 13252   opcode(0x70);
 13253   ins_encode( JccShort( cop, labl) );
 13254   ins_pipe( pipe_jcc );
 13255   ins_pc_relative(1);
 13256   ins_short_branch(1);
 13257 %}
 13259 instruct jmpLoopEndUCF_short(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
 13260   match(CountedLoopEnd cop cmp);
 13261   effect(USE labl);
 13263   ins_cost(300);
 13264   format %{ "J$cop,us $labl\t# Loop end" %}
 13265   size(2);
 13266   opcode(0x70);
 13267   ins_encode( JccShort( cop, labl) );
 13268   ins_pipe( pipe_jcc );
 13269   ins_pc_relative(1);
 13270   ins_short_branch(1);
 13271 %}
 13273 // Jump Direct Conditional - using unsigned comparison
 13274 instruct jmpConU_short(cmpOpU cop, eFlagsRegU cmp, label labl) %{
 13275   match(If cop cmp);
 13276   effect(USE labl);
 13278   ins_cost(300);
 13279   format %{ "J$cop,us $labl" %}
 13280   size(2);
 13281   opcode(0x70);
 13282   ins_encode( JccShort( cop, labl) );
 13283   ins_pipe( pipe_jcc );
 13284   ins_pc_relative(1);
 13285   ins_short_branch(1);
 13286 %}
 13288 instruct jmpConUCF_short(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
 13289   match(If cop cmp);
 13290   effect(USE labl);
 13292   ins_cost(300);
 13293   format %{ "J$cop,us $labl" %}
 13294   size(2);
 13295   opcode(0x70);
 13296   ins_encode( JccShort( cop, labl) );
 13297   ins_pipe( pipe_jcc );
 13298   ins_pc_relative(1);
 13299   ins_short_branch(1);
 13300 %}
 13302 instruct jmpConUCF2_short(cmpOpUCF2 cop, eFlagsRegUCF cmp, label labl) %{
 13303   match(If cop cmp);
 13304   effect(USE labl);
 13306   ins_cost(300);
 13307   format %{ $$template
 13308     if ($cop$$cmpcode == Assembler::notEqual) {
 13309       $$emit$$"JP,u,s   $labl\n\t"
 13310       $$emit$$"J$cop,u,s   $labl"
 13311     } else {
 13312       $$emit$$"JP,u,s   done\n\t"
 13313       $$emit$$"J$cop,u,s  $labl\n\t"
 13314       $$emit$$"done:"
 13316   %}
 13317   size(4);
 13318   opcode(0x70);
 13319   ins_encode %{
 13320     Label* l = $labl$$label;
 13321     emit_cc(cbuf, $primary, Assembler::parity);
 13322     int parity_disp = -1;
 13323     if ($cop$$cmpcode == Assembler::notEqual) {
 13324       parity_disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
 13325     } else if ($cop$$cmpcode == Assembler::equal) {
 13326       parity_disp = 2;
 13327     } else {
 13328       ShouldNotReachHere();
 13330     emit_d8(cbuf, parity_disp);
 13331     emit_cc(cbuf, $primary, $cop$$cmpcode);
 13332     int disp = l ? (l->loc_pos() - (cbuf.insts_size() + 1)) : 0;
 13333     emit_d8(cbuf, disp);
 13334     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
 13335     assert(-128 <= parity_disp && parity_disp <= 127, "Displacement too large for short jmp");
 13336   %}
 13337   ins_pipe(pipe_jcc);
 13338   ins_pc_relative(1);
 13339   ins_short_branch(1);
 13340 %}
 13342 // ============================================================================
 13343 // Long Compare
 13344 //
 13345 // Currently we hold longs in 2 registers.  Comparing such values efficiently
 13346 // is tricky.  The flavor of compare used depends on whether we are testing
 13347 // for LT, LE, or EQ.  For a simple LT test we can check just the sign bit.
 13348 // The GE test is the negated LT test.  The LE test can be had by commuting
 13349 // the operands (yielding a GE test) and then negating; negate again for the
 13350 // GT test.  The EQ test is done by ORcc'ing the high and low halves, and the
 13351 // NE test is negated from that.
 13353 // Due to a shortcoming in the ADLC, it mixes up expressions like:
 13354 // (foo (CmpI (CmpL X Y) 0)) and (bar (CmpI (CmpL X 0L) 0)).  Note the
 13355 // difference between 'Y' and '0L'.  The tree-matches for the CmpI sections
 13356 // are collapsed internally in the ADLC's dfa-gen code.  The match for
 13357 // (CmpI (CmpL X Y) 0) is silently replaced with (CmpI (CmpL X 0L) 0) and the
 13358 // foo match ends up with the wrong leaf.  One fix is to not match both
 13359 // reg-reg and reg-zero forms of long-compare.  This is unfortunate because
 13360 // both forms beat the trinary form of long-compare and both are very useful
 13361 // on Intel which has so few registers.
 13363 // Manifest a CmpL result in an integer register.  Very painful.
 13364 // This is the test to avoid.
 13365 instruct cmpL3_reg_reg(eSIRegI dst, eRegL src1, eRegL src2, eFlagsReg flags ) %{
 13366   match(Set dst (CmpL3 src1 src2));
 13367   effect( KILL flags );
 13368   ins_cost(1000);
 13369   format %{ "XOR    $dst,$dst\n\t"
 13370             "CMP    $src1.hi,$src2.hi\n\t"
 13371             "JLT,s  m_one\n\t"
 13372             "JGT,s  p_one\n\t"
 13373             "CMP    $src1.lo,$src2.lo\n\t"
 13374             "JB,s   m_one\n\t"
 13375             "JEQ,s  done\n"
 13376     "p_one:\tINC    $dst\n\t"
 13377             "JMP,s  done\n"
 13378     "m_one:\tDEC    $dst\n"
 13379      "done:" %}
 13380   ins_encode %{
 13381     Label p_one, m_one, done;
 13382     __ xorptr($dst$$Register, $dst$$Register);
 13383     __ cmpl(HIGH_FROM_LOW($src1$$Register), HIGH_FROM_LOW($src2$$Register));
 13384     __ jccb(Assembler::less,    m_one);
 13385     __ jccb(Assembler::greater, p_one);
 13386     __ cmpl($src1$$Register, $src2$$Register);
 13387     __ jccb(Assembler::below,   m_one);
 13388     __ jccb(Assembler::equal,   done);
 13389     __ bind(p_one);
 13390     __ incrementl($dst$$Register);
 13391     __ jmpb(done);
 13392     __ bind(m_one);
 13393     __ decrementl($dst$$Register);
 13394     __ bind(done);
 13395   %}
 13396   ins_pipe( pipe_slow );
 13397 %}
 13399 //======
 13400 // Manifest a CmpL result in the normal flags.  Only good for LT or GE
 13401 // compares.  Can be used for LE or GT compares by reversing arguments.
 13402 // NOT GOOD FOR EQ/NE tests.
 13403 instruct cmpL_zero_flags_LTGE( flagsReg_long_LTGE flags, eRegL src, immL0 zero ) %{
 13404   match( Set flags (CmpL src zero ));
 13405   ins_cost(100);
 13406   format %{ "TEST   $src.hi,$src.hi" %}
 13407   opcode(0x85);
 13408   ins_encode( OpcP, RegReg_Hi2( src, src ) );
 13409   ins_pipe( ialu_cr_reg_reg );
 13410 %}
 13412 // Manifest a CmpL result in the normal flags.  Only good for LT or GE
 13413 // compares.  Can be used for LE or GT compares by reversing arguments.
 13414 // NOT GOOD FOR EQ/NE tests.
 13415 instruct cmpL_reg_flags_LTGE( flagsReg_long_LTGE flags, eRegL src1, eRegL src2, eRegI tmp ) %{
 13416   match( Set flags (CmpL src1 src2 ));
 13417   effect( TEMP tmp );
 13418   ins_cost(300);
 13419   format %{ "CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits\n\t"
 13420             "MOV    $tmp,$src1.hi\n\t"
 13421             "SBB    $tmp,$src2.hi\t! Compute flags for long compare" %}
 13422   ins_encode( long_cmp_flags2( src1, src2, tmp ) );
 13423   ins_pipe( ialu_cr_reg_reg );
 13424 %}
 13426 // Long compares reg < zero/req OR reg >= zero/req.
 13427 // Just a wrapper for a normal branch, plus the predicate test.
 13428 instruct cmpL_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, label labl) %{
 13429   match(If cmp flags);
 13430   effect(USE labl);
 13431   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
 13432   expand %{
 13433     jmpCon(cmp,flags,labl);    // JLT or JGE...
 13434   %}
 13435 %}
 13437 // Compare 2 longs and CMOVE longs.
 13438 instruct cmovLL_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegL dst, eRegL src) %{
 13439   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
 13440   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
 13441   ins_cost(400);
 13442   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
 13443             "CMOV$cmp $dst.hi,$src.hi" %}
 13444   opcode(0x0F,0x40);
 13445   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
 13446   ins_pipe( pipe_cmov_reg_long );
 13447 %}
 13449 instruct cmovLL_mem_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegL dst, load_long_memory src) %{
 13450   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
 13451   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
 13452   ins_cost(500);
 13453   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
 13454             "CMOV$cmp $dst.hi,$src.hi" %}
 13455   opcode(0x0F,0x40);
 13456   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
 13457   ins_pipe( pipe_cmov_reg_long );
 13458 %}
 13460 // Compare 2 longs and CMOVE ints.
 13461 instruct cmovII_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegI dst, eRegI src) %{
 13462   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
 13463   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
 13464   ins_cost(200);
 13465   format %{ "CMOV$cmp $dst,$src" %}
 13466   opcode(0x0F,0x40);
 13467   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
 13468   ins_pipe( pipe_cmov_reg );
 13469 %}
 13471 instruct cmovII_mem_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegI dst, memory src) %{
 13472   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
 13473   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
 13474   ins_cost(250);
 13475   format %{ "CMOV$cmp $dst,$src" %}
 13476   opcode(0x0F,0x40);
 13477   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
 13478   ins_pipe( pipe_cmov_mem );
 13479 %}
 13481 // Compare 2 longs and CMOVE ints.
 13482 instruct cmovPP_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegP dst, eRegP src) %{
 13483   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
 13484   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
 13485   ins_cost(200);
 13486   format %{ "CMOV$cmp $dst,$src" %}
 13487   opcode(0x0F,0x40);
 13488   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
 13489   ins_pipe( pipe_cmov_reg );
 13490 %}
 13492 // Compare 2 longs and CMOVE doubles
 13493 instruct cmovDD_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regD dst, regD src) %{
 13494   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
 13495   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
 13496   ins_cost(200);
 13497   expand %{
 13498     fcmovD_regS(cmp,flags,dst,src);
 13499   %}
 13500 %}
 13502 // Compare 2 longs and CMOVE doubles
 13503 instruct cmovXDD_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regXD dst, regXD src) %{
 13504   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
 13505   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
 13506   ins_cost(200);
 13507   expand %{
 13508     fcmovXD_regS(cmp,flags,dst,src);
 13509   %}
 13510 %}
 13512 instruct cmovFF_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regF dst, regF src) %{
 13513   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
 13514   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
 13515   ins_cost(200);
 13516   expand %{
 13517     fcmovF_regS(cmp,flags,dst,src);
 13518   %}
 13519 %}
 13521 instruct cmovXX_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regX dst, regX src) %{
 13522   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
 13523   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
 13524   ins_cost(200);
 13525   expand %{
 13526     fcmovX_regS(cmp,flags,dst,src);
 13527   %}
 13528 %}
 13530 //======
 13531 // Manifest a CmpL result in the normal flags.  Only good for EQ/NE compares.
 13532 instruct cmpL_zero_flags_EQNE( flagsReg_long_EQNE flags, eRegL src, immL0 zero, eRegI tmp ) %{
 13533   match( Set flags (CmpL src zero ));
 13534   effect(TEMP tmp);
 13535   ins_cost(200);
 13536   format %{ "MOV    $tmp,$src.lo\n\t"
 13537             "OR     $tmp,$src.hi\t! Long is EQ/NE 0?" %}
 13538   ins_encode( long_cmp_flags0( src, tmp ) );
 13539   ins_pipe( ialu_reg_reg_long );
 13540 %}
 13542 // Manifest a CmpL result in the normal flags.  Only good for EQ/NE compares.
 13543 instruct cmpL_reg_flags_EQNE( flagsReg_long_EQNE flags, eRegL src1, eRegL src2 ) %{
 13544   match( Set flags (CmpL src1 src2 ));
 13545   ins_cost(200+300);
 13546   format %{ "CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits\n\t"
 13547             "JNE,s  skip\n\t"
 13548             "CMP    $src1.hi,$src2.hi\n\t"
 13549      "skip:\t" %}
 13550   ins_encode( long_cmp_flags1( src1, src2 ) );
 13551   ins_pipe( ialu_cr_reg_reg );
 13552 %}
 13554 // Long compare reg == zero/reg OR reg != zero/reg
 13555 // Just a wrapper for a normal branch, plus the predicate test.
 13556 instruct cmpL_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, label labl) %{
 13557   match(If cmp flags);
 13558   effect(USE labl);
 13559   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
 13560   expand %{
 13561     jmpCon(cmp,flags,labl);    // JEQ or JNE...
 13562   %}
 13563 %}
 13565 // Compare 2 longs and CMOVE longs.
 13566 instruct cmovLL_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegL dst, eRegL src) %{
 13567   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
 13568   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
 13569   ins_cost(400);
 13570   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
 13571             "CMOV$cmp $dst.hi,$src.hi" %}
 13572   opcode(0x0F,0x40);
 13573   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
 13574   ins_pipe( pipe_cmov_reg_long );
 13575 %}
 13577 instruct cmovLL_mem_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegL dst, load_long_memory src) %{
 13578   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
 13579   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
 13580   ins_cost(500);
 13581   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
 13582             "CMOV$cmp $dst.hi,$src.hi" %}
 13583   opcode(0x0F,0x40);
 13584   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
 13585   ins_pipe( pipe_cmov_reg_long );
 13586 %}
 13588 // Compare 2 longs and CMOVE ints.
 13589 instruct cmovII_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegI dst, eRegI src) %{
 13590   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
 13591   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
 13592   ins_cost(200);
 13593   format %{ "CMOV$cmp $dst,$src" %}
 13594   opcode(0x0F,0x40);
 13595   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
 13596   ins_pipe( pipe_cmov_reg );
 13597 %}
 13599 instruct cmovII_mem_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegI dst, memory src) %{
 13600   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
 13601   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
 13602   ins_cost(250);
 13603   format %{ "CMOV$cmp $dst,$src" %}
 13604   opcode(0x0F,0x40);
 13605   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
 13606   ins_pipe( pipe_cmov_mem );
 13607 %}
 13609 // Compare 2 longs and CMOVE ints.
 13610 instruct cmovPP_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegP dst, eRegP src) %{
 13611   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
 13612   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
 13613   ins_cost(200);
 13614   format %{ "CMOV$cmp $dst,$src" %}
 13615   opcode(0x0F,0x40);
 13616   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
 13617   ins_pipe( pipe_cmov_reg );
 13618 %}
 13620 // Compare 2 longs and CMOVE doubles
 13621 instruct cmovDD_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regD dst, regD src) %{
 13622   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
 13623   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
 13624   ins_cost(200);
 13625   expand %{
 13626     fcmovD_regS(cmp,flags,dst,src);
 13627   %}
 13628 %}
 13630 // Compare 2 longs and CMOVE doubles
 13631 instruct cmovXDD_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regXD dst, regXD src) %{
 13632   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
 13633   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
 13634   ins_cost(200);
 13635   expand %{
 13636     fcmovXD_regS(cmp,flags,dst,src);
 13637   %}
 13638 %}
 13640 instruct cmovFF_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regF dst, regF src) %{
 13641   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
 13642   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
 13643   ins_cost(200);
 13644   expand %{
 13645     fcmovF_regS(cmp,flags,dst,src);
 13646   %}
 13647 %}
 13649 instruct cmovXX_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regX dst, regX src) %{
 13650   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
 13651   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
 13652   ins_cost(200);
 13653   expand %{
 13654     fcmovX_regS(cmp,flags,dst,src);
 13655   %}
 13656 %}
 13658 //======
 13659 // Manifest a CmpL result in the normal flags.  Only good for LE or GT compares.
 13660 // Same as cmpL_reg_flags_LEGT except must negate src
 13661 instruct cmpL_zero_flags_LEGT( flagsReg_long_LEGT flags, eRegL src, immL0 zero, eRegI tmp ) %{
 13662   match( Set flags (CmpL src zero ));
 13663   effect( TEMP tmp );
 13664   ins_cost(300);
 13665   format %{ "XOR    $tmp,$tmp\t# Long compare for -$src < 0, use commuted test\n\t"
 13666             "CMP    $tmp,$src.lo\n\t"
 13667             "SBB    $tmp,$src.hi\n\t" %}
 13668   ins_encode( long_cmp_flags3(src, tmp) );
 13669   ins_pipe( ialu_reg_reg_long );
 13670 %}
 13672 // Manifest a CmpL result in the normal flags.  Only good for LE or GT compares.
 13673 // Same as cmpL_reg_flags_LTGE except operands swapped.  Swapping operands
 13674 // requires a commuted test to get the same result.
 13675 instruct cmpL_reg_flags_LEGT( flagsReg_long_LEGT flags, eRegL src1, eRegL src2, eRegI tmp ) %{
 13676   match( Set flags (CmpL src1 src2 ));
 13677   effect( TEMP tmp );
 13678   ins_cost(300);
 13679   format %{ "CMP    $src2.lo,$src1.lo\t! Long compare, swapped operands, use with commuted test\n\t"
 13680             "MOV    $tmp,$src2.hi\n\t"
 13681             "SBB    $tmp,$src1.hi\t! Compute flags for long compare" %}
 13682   ins_encode( long_cmp_flags2( src2, src1, tmp ) );
 13683   ins_pipe( ialu_cr_reg_reg );
 13684 %}
 13686 // Long compares reg < zero/req OR reg >= zero/req.
 13687 // Just a wrapper for a normal branch, plus the predicate test
 13688 instruct cmpL_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, label labl) %{
 13689   match(If cmp flags);
 13690   effect(USE labl);
 13691   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le );
 13692   ins_cost(300);
 13693   expand %{
 13694     jmpCon(cmp,flags,labl);    // JGT or JLE...
 13695   %}
 13696 %}
 13698 // Compare 2 longs and CMOVE longs.
 13699 instruct cmovLL_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegL dst, eRegL src) %{
 13700   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
 13701   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
 13702   ins_cost(400);
 13703   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
 13704             "CMOV$cmp $dst.hi,$src.hi" %}
 13705   opcode(0x0F,0x40);
 13706   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
 13707   ins_pipe( pipe_cmov_reg_long );
 13708 %}
 13710 instruct cmovLL_mem_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegL dst, load_long_memory src) %{
 13711   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
 13712   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
 13713   ins_cost(500);
 13714   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
 13715             "CMOV$cmp $dst.hi,$src.hi+4" %}
 13716   opcode(0x0F,0x40);
 13717   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
 13718   ins_pipe( pipe_cmov_reg_long );
 13719 %}
 13721 // Compare 2 longs and CMOVE ints.
 13722 instruct cmovII_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegI dst, eRegI src) %{
 13723   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
 13724   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
 13725   ins_cost(200);
 13726   format %{ "CMOV$cmp $dst,$src" %}
 13727   opcode(0x0F,0x40);
 13728   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
 13729   ins_pipe( pipe_cmov_reg );
 13730 %}
 13732 instruct cmovII_mem_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegI dst, memory src) %{
 13733   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
 13734   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
 13735   ins_cost(250);
 13736   format %{ "CMOV$cmp $dst,$src" %}
 13737   opcode(0x0F,0x40);
 13738   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
 13739   ins_pipe( pipe_cmov_mem );
 13740 %}
 13742 // Compare 2 longs and CMOVE ptrs.
 13743 instruct cmovPP_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegP dst, eRegP src) %{
 13744   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
 13745   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
 13746   ins_cost(200);
 13747   format %{ "CMOV$cmp $dst,$src" %}
 13748   opcode(0x0F,0x40);
 13749   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
 13750   ins_pipe( pipe_cmov_reg );
 13751 %}
 13753 // Compare 2 longs and CMOVE doubles
 13754 instruct cmovDD_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regD dst, regD src) %{
 13755   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
 13756   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
 13757   ins_cost(200);
 13758   expand %{
 13759     fcmovD_regS(cmp,flags,dst,src);
 13760   %}
 13761 %}
 13763 // Compare 2 longs and CMOVE doubles
 13764 instruct cmovXDD_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regXD dst, regXD src) %{
 13765   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
 13766   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
 13767   ins_cost(200);
 13768   expand %{
 13769     fcmovXD_regS(cmp,flags,dst,src);
 13770   %}
 13771 %}
 13773 instruct cmovFF_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regF dst, regF src) %{
 13774   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
 13775   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
 13776   ins_cost(200);
 13777   expand %{
 13778     fcmovF_regS(cmp,flags,dst,src);
 13779   %}
 13780 %}
 13783 instruct cmovXX_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regX dst, regX src) %{
 13784   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
 13785   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
 13786   ins_cost(200);
 13787   expand %{
 13788     fcmovX_regS(cmp,flags,dst,src);
 13789   %}
 13790 %}
 13793 // ============================================================================
 13794 // Procedure Call/Return Instructions
 13795 // Call Java Static Instruction
 13796 // Note: If this code changes, the corresponding ret_addr_offset() and
 13797 //       compute_padding() functions will have to be adjusted.
 13798 instruct CallStaticJavaDirect(method meth) %{
 13799   match(CallStaticJava);
 13800   predicate(! ((CallStaticJavaNode*)n)->is_method_handle_invoke());
 13801   effect(USE meth);
 13803   ins_cost(300);
 13804   format %{ "CALL,static " %}
 13805   opcode(0xE8); /* E8 cd */
 13806   ins_encode( pre_call_FPU,
 13807               Java_Static_Call( meth ),
 13808               call_epilog,
 13809               post_call_FPU );
 13810   ins_pipe( pipe_slow );
 13811   ins_pc_relative(1);
 13812   ins_alignment(4);
 13813 %}
 13815 // Call Java Static Instruction (method handle version)
 13816 // Note: If this code changes, the corresponding ret_addr_offset() and
 13817 //       compute_padding() functions will have to be adjusted.
 13818 instruct CallStaticJavaHandle(method meth, eBPRegP ebp_mh_SP_save) %{
 13819   match(CallStaticJava);
 13820   predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
 13821   effect(USE meth);
 13822   // EBP is saved by all callees (for interpreter stack correction).
 13823   // We use it here for a similar purpose, in {preserve,restore}_SP.
 13825   ins_cost(300);
 13826   format %{ "CALL,static/MethodHandle " %}
 13827   opcode(0xE8); /* E8 cd */
 13828   ins_encode( pre_call_FPU,
 13829               preserve_SP,
 13830               Java_Static_Call( meth ),
 13831               restore_SP,
 13832               call_epilog,
 13833               post_call_FPU );
 13834   ins_pipe( pipe_slow );
 13835   ins_pc_relative(1);
 13836   ins_alignment(4);
 13837 %}
 13839 // Call Java Dynamic Instruction
 13840 // Note: If this code changes, the corresponding ret_addr_offset() and
 13841 //       compute_padding() functions will have to be adjusted.
 13842 instruct CallDynamicJavaDirect(method meth) %{
 13843   match(CallDynamicJava);
 13844   effect(USE meth);
 13846   ins_cost(300);
 13847   format %{ "MOV    EAX,(oop)-1\n\t"
 13848             "CALL,dynamic" %}
 13849   opcode(0xE8); /* E8 cd */
 13850   ins_encode( pre_call_FPU,
 13851               Java_Dynamic_Call( meth ),
 13852               call_epilog,
 13853               post_call_FPU );
 13854   ins_pipe( pipe_slow );
 13855   ins_pc_relative(1);
 13856   ins_alignment(4);
 13857 %}
 13859 // Call Runtime Instruction
 13860 instruct CallRuntimeDirect(method meth) %{
 13861   match(CallRuntime );
 13862   effect(USE meth);
 13864   ins_cost(300);
 13865   format %{ "CALL,runtime " %}
 13866   opcode(0xE8); /* E8 cd */
 13867   // Use FFREEs to clear entries in float stack
 13868   ins_encode( pre_call_FPU,
 13869               FFree_Float_Stack_All,
 13870               Java_To_Runtime( meth ),
 13871               post_call_FPU );
 13872   ins_pipe( pipe_slow );
 13873   ins_pc_relative(1);
 13874 %}
 13876 // Call runtime without safepoint
 13877 instruct CallLeafDirect(method meth) %{
 13878   match(CallLeaf);
 13879   effect(USE meth);
 13881   ins_cost(300);
 13882   format %{ "CALL_LEAF,runtime " %}
 13883   opcode(0xE8); /* E8 cd */
 13884   ins_encode( pre_call_FPU,
 13885               FFree_Float_Stack_All,
 13886               Java_To_Runtime( meth ),
 13887               Verify_FPU_For_Leaf, post_call_FPU );
 13888   ins_pipe( pipe_slow );
 13889   ins_pc_relative(1);
 13890 %}
 13892 instruct CallLeafNoFPDirect(method meth) %{
 13893   match(CallLeafNoFP);
 13894   effect(USE meth);
 13896   ins_cost(300);
 13897   format %{ "CALL_LEAF_NOFP,runtime " %}
 13898   opcode(0xE8); /* E8 cd */
 13899   ins_encode(Java_To_Runtime(meth));
 13900   ins_pipe( pipe_slow );
 13901   ins_pc_relative(1);
 13902 %}
 13905 // Return Instruction
 13906 // Remove the return address & jump to it.
 13907 instruct Ret() %{
 13908   match(Return);
 13909   format %{ "RET" %}
 13910   opcode(0xC3);
 13911   ins_encode(OpcP);
 13912   ins_pipe( pipe_jmp );
 13913 %}
 13915 // Tail Call; Jump from runtime stub to Java code.
 13916 // Also known as an 'interprocedural jump'.
 13917 // Target of jump will eventually return to caller.
 13918 // TailJump below removes the return address.
 13919 instruct TailCalljmpInd(eRegP_no_EBP jump_target, eBXRegP method_oop) %{
 13920   match(TailCall jump_target method_oop );
 13921   ins_cost(300);
 13922   format %{ "JMP    $jump_target \t# EBX holds method oop" %}
 13923   opcode(0xFF, 0x4);  /* Opcode FF /4 */
 13924   ins_encode( OpcP, RegOpc(jump_target) );
 13925   ins_pipe( pipe_jmp );
 13926 %}
 13929 // Tail Jump; remove the return address; jump to target.
 13930 // TailCall above leaves the return address around.
 13931 instruct tailjmpInd(eRegP_no_EBP jump_target, eAXRegP ex_oop) %{
 13932   match( TailJump jump_target ex_oop );
 13933   ins_cost(300);
 13934   format %{ "POP    EDX\t# pop return address into dummy\n\t"
 13935             "JMP    $jump_target " %}
 13936   opcode(0xFF, 0x4);  /* Opcode FF /4 */
 13937   ins_encode( enc_pop_rdx,
 13938               OpcP, RegOpc(jump_target) );
 13939   ins_pipe( pipe_jmp );
 13940 %}
 13942 // Create exception oop: created by stack-crawling runtime code.
 13943 // Created exception is now available to this handler, and is setup
 13944 // just prior to jumping to this handler.  No code emitted.
 13945 instruct CreateException( eAXRegP ex_oop )
 13946 %{
 13947   match(Set ex_oop (CreateEx));
 13949   size(0);
 13950   // use the following format syntax
 13951   format %{ "# exception oop is in EAX; no code emitted" %}
 13952   ins_encode();
 13953   ins_pipe( empty );
 13954 %}
 13957 // Rethrow exception:
 13958 // The exception oop will come in the first argument position.
 13959 // Then JUMP (not call) to the rethrow stub code.
 13960 instruct RethrowException()
 13961 %{
 13962   match(Rethrow);
 13964   // use the following format syntax
 13965   format %{ "JMP    rethrow_stub" %}
 13966   ins_encode(enc_rethrow);
 13967   ins_pipe( pipe_jmp );
 13968 %}
 13970 // inlined locking and unlocking
 13973 instruct cmpFastLock( eFlagsReg cr, eRegP object, eRegP box, eAXRegI tmp, eRegP scr) %{
 13974   match( Set cr (FastLock object box) );
 13975   effect( TEMP tmp, TEMP scr );
 13976   ins_cost(300);
 13977   format %{ "FASTLOCK $object, $box KILLS $tmp,$scr" %}
 13978   ins_encode( Fast_Lock(object,box,tmp,scr) );
 13979   ins_pipe( pipe_slow );
 13980   ins_pc_relative(1);
 13981 %}
 13983 instruct cmpFastUnlock( eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{
 13984   match( Set cr (FastUnlock object box) );
 13985   effect( TEMP tmp );
 13986   ins_cost(300);
 13987   format %{ "FASTUNLOCK $object, $box, $tmp" %}
 13988   ins_encode( Fast_Unlock(object,box,tmp) );
 13989   ins_pipe( pipe_slow );
 13990   ins_pc_relative(1);
 13991 %}
 13995 // ============================================================================
 13996 // Safepoint Instruction
 13997 instruct safePoint_poll(eFlagsReg cr) %{
 13998   match(SafePoint);
 13999   effect(KILL cr);
 14001   // TODO-FIXME: we currently poll at offset 0 of the safepoint polling page.
 14002   // On SPARC that might be acceptable as we can generate the address with
 14003   // just a sethi, saving an or.  By polling at offset 0 we can end up
 14004   // putting additional pressure on the index-0 in the D$.  Because of
 14005   // alignment (just like the situation at hand) the lower indices tend
 14006   // to see more traffic.  It'd be better to change the polling address
 14007   // to offset 0 of the last $line in the polling page.
 14009   format %{ "TSTL   #polladdr,EAX\t! Safepoint: poll for GC" %}
 14010   ins_cost(125);
 14011   size(6) ;
 14012   ins_encode( Safepoint_Poll() );
 14013   ins_pipe( ialu_reg_mem );
 14014 %}
 14016 //----------PEEPHOLE RULES-----------------------------------------------------
 14017 // These must follow all instruction definitions as they use the names
 14018 // defined in the instructions definitions.
 14019 //
 14020 // peepmatch ( root_instr_name [preceding_instruction]* );
 14021 //
 14022 // peepconstraint %{
 14023 // (instruction_number.operand_name relational_op instruction_number.operand_name
 14024 //  [, ...] );
 14025 // // instruction numbers are zero-based using left to right order in peepmatch
 14026 //
 14027 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
 14028 // // provide an instruction_number.operand_name for each operand that appears
 14029 // // in the replacement instruction's match rule
 14030 //
 14031 // ---------VM FLAGS---------------------------------------------------------
 14032 //
 14033 // All peephole optimizations can be turned off using -XX:-OptoPeephole
 14034 //
 14035 // Each peephole rule is given an identifying number starting with zero and
 14036 // increasing by one in the order seen by the parser.  An individual peephole
 14037 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
 14038 // on the command-line.
 14039 //
 14040 // ---------CURRENT LIMITATIONS----------------------------------------------
 14041 //
 14042 // Only match adjacent instructions in same basic block
 14043 // Only equality constraints
 14044 // Only constraints between operands, not (0.dest_reg == EAX_enc)
 14045 // Only one replacement instruction
 14046 //
 14047 // ---------EXAMPLE----------------------------------------------------------
 14048 //
 14049 // // pertinent parts of existing instructions in architecture description
 14050 // instruct movI(eRegI dst, eRegI src) %{
 14051 //   match(Set dst (CopyI src));
 14052 // %}
 14053 //
 14054 // instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
 14055 //   match(Set dst (AddI dst src));
 14056 //   effect(KILL cr);
 14057 // %}
 14058 //
 14059 // // Change (inc mov) to lea
 14060 // peephole %{
 14061 //   // increment preceeded by register-register move
 14062 //   peepmatch ( incI_eReg movI );
 14063 //   // require that the destination register of the increment
 14064 //   // match the destination register of the move
 14065 //   peepconstraint ( 0.dst == 1.dst );
 14066 //   // construct a replacement instruction that sets
 14067 //   // the destination to ( move's source register + one )
 14068 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
 14069 // %}
 14070 //
 14071 // Implementation no longer uses movX instructions since
 14072 // machine-independent system no longer uses CopyX nodes.
 14073 //
 14074 // peephole %{
 14075 //   peepmatch ( incI_eReg movI );
 14076 //   peepconstraint ( 0.dst == 1.dst );
 14077 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
 14078 // %}
 14079 //
 14080 // peephole %{
 14081 //   peepmatch ( decI_eReg movI );
 14082 //   peepconstraint ( 0.dst == 1.dst );
 14083 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
 14084 // %}
 14085 //
 14086 // peephole %{
 14087 //   peepmatch ( addI_eReg_imm movI );
 14088 //   peepconstraint ( 0.dst == 1.dst );
 14089 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
 14090 // %}
 14091 //
 14092 // peephole %{
 14093 //   peepmatch ( addP_eReg_imm movP );
 14094 //   peepconstraint ( 0.dst == 1.dst );
 14095 //   peepreplace ( leaP_eReg_immI( 0.dst 1.src 0.src ) );
 14096 // %}
 14098 // // Change load of spilled value to only a spill
 14099 // instruct storeI(memory mem, eRegI src) %{
 14100 //   match(Set mem (StoreI mem src));
 14101 // %}
 14102 //
 14103 // instruct loadI(eRegI dst, memory mem) %{
 14104 //   match(Set dst (LoadI mem));
 14105 // %}
 14106 //
 14107 peephole %{
 14108   peepmatch ( loadI storeI );
 14109   peepconstraint ( 1.src == 0.dst, 1.mem == 0.mem );
 14110   peepreplace ( storeI( 1.mem 1.mem 1.src ) );
 14111 %}
 14113 //----------SMARTSPILL RULES---------------------------------------------------
 14114 // These must follow all instruction definitions as they use the names
 14115 // defined in the instructions definitions.

mercurial