Mercurial > jdk8-mips64-public > hotspot / file revision

 //

 // Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.

 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 //

 // This code is free software; you can redistribute it and/or modify it

 // under the terms of the GNU General Public License version 2 only, as

 // published by the Free Software Foundation.

 //

 // This code is distributed in the hope that it will be useful, but WITHOUT

 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 // version 2 for more details (a copy is included in the LICENSE file that

 // accompanied this code).

 //

 // You should have received a copy of the GNU General Public License version

 // 2 along with this work; if not, write to the Free Software Foundation,

 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 //

 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 // or visit www.oracle.com if you need additional information or have any

 // questions.

 //

 //

 // X86 Architecture Description File

 //----------REGISTER DEFINITION BLOCK------------------------------------------

 // This information is used by the matcher and the register allocator to

 // describe individual registers and classes of registers within the target

 // archtecture.

 register %{

 //----------Architecture Description Register Definitions----------------------

 // General Registers

 // "reg_def"  name ( register save type, C convention save type,

 //                   ideal register type, encoding );

 // Register Save Types:

 //

 // NS  = No-Save:       The register allocator assumes that these registers

 //                      can be used without saving upon entry to the method, &

 //                      that they do not need to be saved at call sites.

 //

 // SOC = Save-On-Call:  The register allocator assumes that these registers

 //                      can be used without saving upon entry to the method,

 //                      but that they must be saved at call sites.

 //

 // SOE = Save-On-Entry: The register allocator assumes that these registers

 //                      must be saved before using them upon entry to the

 //                      method, but they do not need to be saved at call

 //                      sites.

 //

 // AS  = Always-Save:   The register allocator assumes that these registers

 //                      must be saved before using them upon entry to the

 //                      method, & that they must be saved at call sites.

 //

 // Ideal Register Type is used to determine how to save & restore a

 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get

 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.

 //

 // The encoding number is the actual bit-pattern placed into the opcodes.

 // General Registers

 // Previously set EBX, ESI, and EDI as save-on-entry for java code

 // Turn off SOE in java-code due to frequent use of uncommon-traps.

 // Now that allocator is better, turn on ESI and EDI as SOE registers.

 reg_def EBX(SOC, SOE, Op_RegI, 3, rbx->as_VMReg());

 reg_def ECX(SOC, SOC, Op_RegI, 1, rcx->as_VMReg());

 reg_def ESI(SOC, SOE, Op_RegI, 6, rsi->as_VMReg());

 reg_def EDI(SOC, SOE, Op_RegI, 7, rdi->as_VMReg());

 // now that adapter frames are gone EBP is always saved and restored by the prolog/epilog code

 reg_def EBP(NS, SOE, Op_RegI, 5, rbp->as_VMReg());

 reg_def EDX(SOC, SOC, Op_RegI, 2, rdx->as_VMReg());

 reg_def EAX(SOC, SOC, Op_RegI, 0, rax->as_VMReg());

 reg_def ESP( NS,  NS, Op_RegI, 4, rsp->as_VMReg());

 // Special Registers

 reg_def EFLAGS(SOC, SOC, 0, 8, VMRegImpl::Bad());

 // Float registers.  We treat TOS/FPR0 special.  It is invisible to the

 // allocator, and only shows up in the encodings.

 reg_def FPR0L( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());

 reg_def FPR0H( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());

 // Ok so here's the trick FPR1 is really st(0) except in the midst

 // of emission of assembly for a machnode. During the emission the fpu stack

 // is pushed making FPR1 == st(1) temporarily. However at any safepoint

 // the stack will not have this element so FPR1 == st(0) from the

 // oopMap viewpoint. This same weirdness with numbering causes

 // instruction encoding to have to play games with the register

 // encode to correct for this 0/1 issue. See MachSpillCopyNode::implementation

 // where it does flt->flt moves to see an example

 //

 reg_def FPR1L( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg());

 reg_def FPR1H( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg()->next());

 reg_def FPR2L( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg());

 reg_def FPR2H( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg()->next());

 reg_def FPR3L( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg());

 reg_def FPR3H( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg()->next());

 reg_def FPR4L( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg());

 reg_def FPR4H( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg()->next());

 reg_def FPR5L( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg());

 reg_def FPR5H( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg()->next());

 reg_def FPR6L( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg());

 reg_def FPR6H( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg()->next());

 reg_def FPR7L( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg());

 reg_def FPR7H( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg()->next());

 // XMM registers.  128-bit registers or 4 words each, labeled a-d.

 // Word a in each register holds a Float, words ab hold a Double.

 // We currently do not use the SIMD capabilities, so registers cd

 // are unused at the moment.

 reg_def XMM0a( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());

 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next());

 reg_def XMM1a( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());

 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next());

 reg_def XMM2a( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());

 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next());

 reg_def XMM3a( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());

 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next());

 reg_def XMM4a( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());

 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next());

 reg_def XMM5a( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());

 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next());

 reg_def XMM6a( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());

 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next());

 reg_def XMM7a( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());

 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next());

 // Specify priority of register selection within phases of register

 // allocation.  Highest priority is first.  A useful heuristic is to

 // give registers a low priority when they are required by machine

 // instructions, like EAX and EDX.  Registers which are used as

 // pairs must fall on an even boundary (witness the FPR#L's in this list).

 // For the Intel integer registers, the equivalent Long pairs are

 // EDX:EAX, EBX:ECX, and EDI:EBP.

 alloc_class chunk0( ECX,   EBX,   EBP,   EDI,   EAX,   EDX,   ESI, ESP,

                     FPR0L, FPR0H, FPR1L, FPR1H, FPR2L, FPR2H,

                     FPR3L, FPR3H, FPR4L, FPR4H, FPR5L, FPR5H,

                     FPR6L, FPR6H, FPR7L, FPR7H );

 alloc_class chunk1( XMM0a, XMM0b,

                     XMM1a, XMM1b,

                     XMM2a, XMM2b,

                     XMM3a, XMM3b,

                     XMM4a, XMM4b,

                     XMM5a, XMM5b,

                     XMM6a, XMM6b,

                     XMM7a, XMM7b, EFLAGS);

 //----------Architecture Description Register Classes--------------------------

 // Several register classes are automatically defined based upon information in

 // this architecture description.

 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )

 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )

 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )

 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )

 //

 // Class for all registers

 reg_class any_reg(EAX, EDX, EBP, EDI, ESI, ECX, EBX, ESP);

 // Class for general registers

 reg_class e_reg(EAX, EDX, EBP, EDI, ESI, ECX, EBX);

 // Class for general registers which may be used for implicit null checks on win95

 // Also safe for use by tailjump. We don't want to allocate in rbp,

 reg_class e_reg_no_rbp(EAX, EDX, EDI, ESI, ECX, EBX);

 // Class of "X" registers

 reg_class x_reg(EBX, ECX, EDX, EAX);

 // Class of registers that can appear in an address with no offset.

 // EBP and ESP require an extra instruction byte for zero offset.

 // Used in fast-unlock

 reg_class p_reg(EDX, EDI, ESI, EBX);

 // Class for general registers not including ECX

 reg_class ncx_reg(EAX, EDX, EBP, EDI, ESI, EBX);

 // Class for general registers not including EAX

 reg_class nax_reg(EDX, EDI, ESI, ECX, EBX);

 // Class for general registers not including EAX or EBX.

 reg_class nabx_reg(EDX, EDI, ESI, ECX, EBP);

 // Class of EAX (for multiply and divide operations)

 reg_class eax_reg(EAX);

 // Class of EBX (for atomic add)

 reg_class ebx_reg(EBX);

 // Class of ECX (for shift and JCXZ operations and cmpLTMask)

 reg_class ecx_reg(ECX);

 // Class of EDX (for multiply and divide operations)

 reg_class edx_reg(EDX);

 // Class of EDI (for synchronization)

 reg_class edi_reg(EDI);

 // Class of ESI (for synchronization)

 reg_class esi_reg(ESI);

 // Singleton class for interpreter's stack pointer

 reg_class ebp_reg(EBP);

 // Singleton class for stack pointer

 reg_class sp_reg(ESP);

 // Singleton class for instruction pointer

 // reg_class ip_reg(EIP);

 // Singleton class for condition codes

 reg_class int_flags(EFLAGS);

 // Class of integer register pairs

 reg_class long_reg( EAX,EDX, ECX,EBX, EBP,EDI );

 // Class of integer register pairs that aligns with calling convention

 reg_class eadx_reg( EAX,EDX );

 reg_class ebcx_reg( ECX,EBX );

 // Not AX or DX, used in divides

 reg_class nadx_reg( EBX,ECX,ESI,EDI,EBP );

 // Floating point registers.  Notice FPR0 is not a choice.

 // FPR0 is not ever allocated; we use clever encodings to fake

 // a 2-address instructions out of Intels FP stack.

 reg_class flt_reg( FPR1L,FPR2L,FPR3L,FPR4L,FPR5L,FPR6L,FPR7L );

 // make a register class for SSE registers

 reg_class xmm_reg(XMM0a, XMM1a, XMM2a, XMM3a, XMM4a, XMM5a, XMM6a, XMM7a);

 // make a double register class for SSE2 registers

 reg_class xdb_reg(XMM0a,XMM0b, XMM1a,XMM1b, XMM2a,XMM2b, XMM3a,XMM3b,

                   XMM4a,XMM4b, XMM5a,XMM5b, XMM6a,XMM6b, XMM7a,XMM7b );

 reg_class dbl_reg( FPR1L,FPR1H, FPR2L,FPR2H, FPR3L,FPR3H,

                    FPR4L,FPR4H, FPR5L,FPR5H, FPR6L,FPR6H,

                    FPR7L,FPR7H );

 reg_class flt_reg0( FPR1L );

 reg_class dbl_reg0( FPR1L,FPR1H );

 reg_class dbl_reg1( FPR2L,FPR2H );

 reg_class dbl_notreg0( FPR2L,FPR2H, FPR3L,FPR3H, FPR4L,FPR4H,

                        FPR5L,FPR5H, FPR6L,FPR6H, FPR7L,FPR7H );

 // XMM6 and XMM7 could be used as temporary registers for long, float and

 // double values for SSE2.

 reg_class xdb_reg6( XMM6a,XMM6b );

 reg_class xdb_reg7( XMM7a,XMM7b );

 %}

 //----------SOURCE BLOCK-------------------------------------------------------

 // This is a block of C++ code which provides values, functions, and

 // definitions necessary in the rest of the architecture description

 source_hpp %{

 // Must be visible to the DFA in dfa_x86_32.cpp

 extern bool is_operand_hi32_zero(Node* n);

 %}

 source %{

 #define   RELOC_IMM32    Assembler::imm_operand

 #define   RELOC_DISP32   Assembler::disp32_operand

 #define __ _masm.

 // How to find the high register of a Long pair, given the low register

 #define   HIGH_FROM_LOW(x) ((x)+2)

 // These masks are used to provide 128-bit aligned bitmasks to the XMM

 // instructions, to allow sign-masking or sign-bit flipping.  They allow

 // fast versions of NegF/NegD and AbsF/AbsD.

 // Note: 'double' and 'long long' have 32-bits alignment on x86.

 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {

   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address

   // of 128-bits operands for SSE instructions.

   jlong *operand = (jlong*)(((uintptr_t)adr)&((uintptr_t)(~0xF)));

   // Store the value to a 128-bits operand.

   operand[0] = lo;

   operand[1] = hi;

   return operand;

 }

 // Buffer for 128-bits masks used by SSE instructions.

 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)

 // Static initialization during VM startup.

 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));

 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));

 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], CONST64(0x8000000080000000), CONST64(0x8000000080000000));

 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], CONST64(0x8000000000000000), CONST64(0x8000000000000000));

 // Offset hacking within calls.

 static int pre_call_FPU_size() {

   if (Compile::current()->in_24_bit_fp_mode())

     return 6; // fldcw

   return 0;

 }

 static int preserve_SP_size() {

   return LP64_ONLY(1 +) 2;  // [rex,] op, rm(reg/reg)

 }

 // !!!!! Special hack to get all type of calls to specify the byte offset

 //       from the start of the call to the point where the return address

 //       will point.

 int MachCallStaticJavaNode::ret_addr_offset() {

   int offset = 5 + pre_call_FPU_size();  // 5 bytes from start of call to where return address points

   if (_method_handle_invoke)

     offset += preserve_SP_size();

   return offset;

 }

 int MachCallDynamicJavaNode::ret_addr_offset() {

   return 10 + pre_call_FPU_size();  // 10 bytes from start of call to where return address points

 }

 static int sizeof_FFree_Float_Stack_All = -1;

 int MachCallRuntimeNode::ret_addr_offset() {

   assert(sizeof_FFree_Float_Stack_All != -1, "must have been emitted already");

   return sizeof_FFree_Float_Stack_All + 5 + pre_call_FPU_size();

 }

 // Indicate if the safepoint node needs the polling page as an input.

 // Since x86 does have absolute addressing, it doesn't.

 bool SafePointNode::needs_polling_address_input() {

   return false;

 }

 //

 // Compute padding required for nodes which need alignment

 //

 // The address of the call instruction needs to be 4-byte aligned to

 // ensure that it does not span a cache line so that it can be patched.

 int CallStaticJavaDirectNode::compute_padding(int current_offset) const {

   current_offset += pre_call_FPU_size();  // skip fldcw, if any

   current_offset += 1;      // skip call opcode byte

   return round_to(current_offset, alignment_required()) - current_offset;

 }

 // The address of the call instruction needs to be 4-byte aligned to

 // ensure that it does not span a cache line so that it can be patched.

 int CallStaticJavaHandleNode::compute_padding(int current_offset) const {

   current_offset += pre_call_FPU_size();  // skip fldcw, if any

   current_offset += preserve_SP_size();   // skip mov rbp, rsp

   current_offset += 1;      // skip call opcode byte

   return round_to(current_offset, alignment_required()) - current_offset;

 }

 // The address of the call instruction needs to be 4-byte aligned to

 // ensure that it does not span a cache line so that it can be patched.

 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const {

   current_offset += pre_call_FPU_size();  // skip fldcw, if any

   current_offset += 5;      // skip MOV instruction

   current_offset += 1;      // skip call opcode byte

   return round_to(current_offset, alignment_required()) - current_offset;

 }

 #ifndef PRODUCT

 void MachBreakpointNode::format( PhaseRegAlloc *, outputStream* st ) const {

   st->print("INT3");

 }

 #endif

 // EMIT_RM()

 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {

   unsigned char c = (unsigned char)((f1 << 6) | (f2 << 3) | f3);

   *(cbuf.code_end()) = c;

   cbuf.set_code_end(cbuf.code_end() + 1);

 }

 // EMIT_CC()

 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {

   unsigned char c = (unsigned char)( f1 | f2 );

   *(cbuf.code_end()) = c;

   cbuf.set_code_end(cbuf.code_end() + 1);

 }

 // EMIT_OPCODE()

 void emit_opcode(CodeBuffer &cbuf, int code) {

   *(cbuf.code_end()) = (unsigned char)code;

   cbuf.set_code_end(cbuf.code_end() + 1);

 }

 // EMIT_OPCODE() w/ relocation information

 void emit_opcode(CodeBuffer &cbuf, int code, relocInfo::relocType reloc, int offset = 0) {

   cbuf.relocate(cbuf.inst_mark() + offset, reloc);

   emit_opcode(cbuf, code);

 }

 // EMIT_D8()

 void emit_d8(CodeBuffer &cbuf, int d8) {

   *(cbuf.code_end()) = (unsigned char)d8;

   cbuf.set_code_end(cbuf.code_end() + 1);

 }

 // EMIT_D16()

 void emit_d16(CodeBuffer &cbuf, int d16) {

   *((short *)(cbuf.code_end())) = d16;

   cbuf.set_code_end(cbuf.code_end() + 2);

 }

 // EMIT_D32()

 void emit_d32(CodeBuffer &cbuf, int d32) {

   *((int *)(cbuf.code_end())) = d32;

   cbuf.set_code_end(cbuf.code_end() + 4);

 }

 // emit 32 bit value and construct relocation entry from relocInfo::relocType

 void emit_d32_reloc(CodeBuffer &cbuf, int d32, relocInfo::relocType reloc,

         int format) {

   cbuf.relocate(cbuf.inst_mark(), reloc, format);

   *((int *)(cbuf.code_end())) = d32;

   cbuf.set_code_end(cbuf.code_end() + 4);

 }

 // emit 32 bit value and construct relocation entry from RelocationHolder

 void emit_d32_reloc(CodeBuffer &cbuf, int d32, RelocationHolder const& rspec,

         int format) {

 #ifdef ASSERT

   if (rspec.reloc()->type() == relocInfo::oop_type && d32 != 0 && d32 != (int)Universe::non_oop_word()) {

     assert(oop(d32)->is_oop() && (ScavengeRootsInCode || !oop(d32)->is_scavengable()), "cannot embed scavengable oops in code");

   }

 #endif

   cbuf.relocate(cbuf.inst_mark(), rspec, format);

   *((int *)(cbuf.code_end())) = d32;

   cbuf.set_code_end(cbuf.code_end() + 4);

 }

 // Access stack slot for load or store

 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp) {

   emit_opcode( cbuf, opcode );               // (e.g., FILD   [ESP+src])

   if( -128 <= disp && disp <= 127 ) {

     emit_rm( cbuf, 0x01, rm_field, ESP_enc );  // R/M byte

     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte

     emit_d8 (cbuf, disp);     // Displacement  // R/M byte

   } else {

     emit_rm( cbuf, 0x02, rm_field, ESP_enc );  // R/M byte

     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte

     emit_d32(cbuf, disp);     // Displacement  // R/M byte

   }

 }

    // eRegI ereg, memory mem) %{    // emit_reg_mem

 void encode_RegMem( CodeBuffer &cbuf, int reg_encoding, int base, int index, int scale, int displace, bool displace_is_oop ) {

   // There is no index & no scale, use form without SIB byte

   if ((index == 0x4) &&

       (scale == 0) && (base != ESP_enc)) {

     // If no displacement, mode is 0x0; unless base is [EBP]

     if ( (displace == 0) && (base != EBP_enc) ) {

       emit_rm(cbuf, 0x0, reg_encoding, base);

     }

     else {                    // If 8-bit displacement, mode 0x1

       if ((displace >= -128) && (displace <= 127)

           && !(displace_is_oop) ) {

         emit_rm(cbuf, 0x1, reg_encoding, base);

         emit_d8(cbuf, displace);

       }

       else {                  // If 32-bit displacement

         if (base == -1) { // Special flag for absolute address

           emit_rm(cbuf, 0x0, reg_encoding, 0x5);

           // (manual lies; no SIB needed here)

           if ( displace_is_oop ) {

             emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);

           } else {

             emit_d32      (cbuf, displace);

           }

         }

         else {                // Normal base + offset

           emit_rm(cbuf, 0x2, reg_encoding, base);

           if ( displace_is_oop ) {

             emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);

           } else {

             emit_d32      (cbuf, displace);

           }

         }

       }

     }

   }

   else {                      // Else, encode with the SIB byte

     // If no displacement, mode is 0x0; unless base is [EBP]

     if (displace == 0 && (base != EBP_enc)) {  // If no displacement

       emit_rm(cbuf, 0x0, reg_encoding, 0x4);

       emit_rm(cbuf, scale, index, base);

     }

     else {                    // If 8-bit displacement, mode 0x1

       if ((displace >= -128) && (displace <= 127)

           && !(displace_is_oop) ) {

         emit_rm(cbuf, 0x1, reg_encoding, 0x4);

         emit_rm(cbuf, scale, index, base);

         emit_d8(cbuf, displace);

       }

       else {                  // If 32-bit displacement

         if (base == 0x04 ) {

           emit_rm(cbuf, 0x2, reg_encoding, 0x4);

           emit_rm(cbuf, scale, index, 0x04);

         } else {

           emit_rm(cbuf, 0x2, reg_encoding, 0x4);

           emit_rm(cbuf, scale, index, base);

         }

         if ( displace_is_oop ) {

           emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);

         } else {

           emit_d32      (cbuf, displace);

         }

       }

     }

   }

 }

 void encode_Copy( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {

   if( dst_encoding == src_encoding ) {

     // reg-reg copy, use an empty encoding

   } else {

     emit_opcode( cbuf, 0x8B );

     emit_rm(cbuf, 0x3, dst_encoding, src_encoding );

   }

 }

 void encode_CopyXD( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {

   if( dst_encoding == src_encoding ) {

     // reg-reg copy, use an empty encoding

   } else {

     MacroAssembler _masm(&cbuf);

     __ movdqa(as_XMMRegister(dst_encoding), as_XMMRegister(src_encoding));

   }

 }

 //=============================================================================

 #ifndef PRODUCT

 void MachPrologNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {

   Compile* C = ra_->C;

   if( C->in_24_bit_fp_mode() ) {

     st->print("FLDCW  24 bit fpu control word");

     st->print_cr(""); st->print("\t");

   }

   int framesize = C->frame_slots() << LogBytesPerInt;

   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");

   // Remove two words for return addr and rbp,

   framesize -= 2*wordSize;

   // Calls to C2R adapters often do not accept exceptional returns.

   // We require that their callers must bang for them.  But be careful, because

   // some VM calls (such as call site linkage) can use several kilobytes of

   // stack.  But the stack safety zone should account for that.

   // See bugs 4446381, 4468289, 4497237.

   if (C->need_stack_bang(framesize)) {

     st->print_cr("# stack bang"); st->print("\t");

   }

   st->print_cr("PUSHL  EBP"); st->print("\t");

   if( VerifyStackAtCalls ) { // Majik cookie to verify stack depth

     st->print("PUSH   0xBADB100D\t# Majik cookie for stack depth check");

     st->print_cr(""); st->print("\t");

     framesize -= wordSize;

   }

   if ((C->in_24_bit_fp_mode() || VerifyStackAtCalls ) && framesize < 128 ) {

     if (framesize) {

       st->print("SUB    ESP,%d\t# Create frame",framesize);

     }

   } else {

     st->print("SUB    ESP,%d\t# Create frame",framesize);

   }

 }

 #endif

 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

   Compile* C = ra_->C;

   if (UseSSE >= 2 && VerifyFPU) {

     MacroAssembler masm(&cbuf);

     masm.verify_FPU(0, "FPU stack must be clean on entry");

   }

   // WARNING: Initial instruction MUST be 5 bytes or longer so that

   // NativeJump::patch_verified_entry will be able to patch out the entry

   // code safely. The fldcw is ok at 6 bytes, the push to verify stack

   // depth is ok at 5 bytes, the frame allocation can be either 3 or

   // 6 bytes. So if we don't do the fldcw or the push then we must

   // use the 6 byte frame allocation even if we have no frame. :-(

   // If method sets FPU control word do it now

   if( C->in_24_bit_fp_mode() ) {

     MacroAssembler masm(&cbuf);

     masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));

   }

   int framesize = C->frame_slots() << LogBytesPerInt;

   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");

   // Remove two words for return addr and rbp,

   framesize -= 2*wordSize;

   // Calls to C2R adapters often do not accept exceptional returns.

   // We require that their callers must bang for them.  But be careful, because

   // some VM calls (such as call site linkage) can use several kilobytes of

   // stack.  But the stack safety zone should account for that.

   // See bugs 4446381, 4468289, 4497237.

   if (C->need_stack_bang(framesize)) {

     MacroAssembler masm(&cbuf);

     masm.generate_stack_overflow_check(framesize);

   }

   // We always push rbp, so that on return to interpreter rbp, will be

   // restored correctly and we can correct the stack.

   emit_opcode(cbuf, 0x50 | EBP_enc);

   if( VerifyStackAtCalls ) { // Majik cookie to verify stack depth

     emit_opcode(cbuf, 0x68); // push 0xbadb100d

     emit_d32(cbuf, 0xbadb100d);

     framesize -= wordSize;

   }

   if ((C->in_24_bit_fp_mode() || VerifyStackAtCalls ) && framesize < 128 ) {

     if (framesize) {

       emit_opcode(cbuf, 0x83);   // sub  SP,#framesize

       emit_rm(cbuf, 0x3, 0x05, ESP_enc);

       emit_d8(cbuf, framesize);

     }

   } else {

     emit_opcode(cbuf, 0x81);   // sub  SP,#framesize

     emit_rm(cbuf, 0x3, 0x05, ESP_enc);

     emit_d32(cbuf, framesize);

   }

   C->set_frame_complete(cbuf.code_end() - cbuf.code_begin());

 #ifdef ASSERT

   if (VerifyStackAtCalls) {

     Label L;

     MacroAssembler masm(&cbuf);

     masm.push(rax);

     masm.mov(rax, rsp);

     masm.andptr(rax, StackAlignmentInBytes-1);

     masm.cmpptr(rax, StackAlignmentInBytes-wordSize);

     masm.pop(rax);

     masm.jcc(Assembler::equal, L);

     masm.stop("Stack is not properly aligned!");

     masm.bind(L);

   }

 #endif

 }

 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {

   return MachNode::size(ra_); // too many variables; just compute it the hard way

 }

 int MachPrologNode::reloc() const {

   return 0; // a large enough number

 }

 //=============================================================================

 #ifndef PRODUCT

 void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {

   Compile *C = ra_->C;

   int framesize = C->frame_slots() << LogBytesPerInt;

   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");

   // Remove two words for return addr and rbp,

   framesize -= 2*wordSize;

   if( C->in_24_bit_fp_mode() ) {

     st->print("FLDCW  standard control word");

     st->cr(); st->print("\t");

   }

   if( framesize ) {

     st->print("ADD    ESP,%d\t# Destroy frame",framesize);

     st->cr(); st->print("\t");

   }

   st->print_cr("POPL   EBP"); st->print("\t");

   if( do_polling() && C->is_method_compilation() ) {

     st->print("TEST   PollPage,EAX\t! Poll Safepoint");

     st->cr(); st->print("\t");

   }

 }

 #endif

 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

   Compile *C = ra_->C;

   // If method set FPU control word, restore to standard control word

   if( C->in_24_bit_fp_mode() ) {

     MacroAssembler masm(&cbuf);

     masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

   }

   int framesize = C->frame_slots() << LogBytesPerInt;

   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");

   // Remove two words for return addr and rbp,

   framesize -= 2*wordSize;

   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here

   if( framesize >= 128 ) {

     emit_opcode(cbuf, 0x81); // add  SP, #framesize

     emit_rm(cbuf, 0x3, 0x00, ESP_enc);

     emit_d32(cbuf, framesize);

   }

   else if( framesize ) {

     emit_opcode(cbuf, 0x83); // add  SP, #framesize

     emit_rm(cbuf, 0x3, 0x00, ESP_enc);

     emit_d8(cbuf, framesize);

   }

   emit_opcode(cbuf, 0x58 | EBP_enc);

   if( do_polling() && C->is_method_compilation() ) {

     cbuf.relocate(cbuf.code_end(), relocInfo::poll_return_type, 0);

     emit_opcode(cbuf,0x85);

     emit_rm(cbuf, 0x0, EAX_enc, 0x5); // EAX

     emit_d32(cbuf, (intptr_t)os::get_polling_page());

   }

 }

 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {

   Compile *C = ra_->C;

   // If method set FPU control word, restore to standard control word

   int size = C->in_24_bit_fp_mode() ? 6 : 0;

   if( do_polling() && C->is_method_compilation() ) size += 6;

   int framesize = C->frame_slots() << LogBytesPerInt;

   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");

   // Remove two words for return addr and rbp,

   framesize -= 2*wordSize;

   size++; // popl rbp,

   if( framesize >= 128 ) {

     size += 6;

   } else {

     size += framesize ? 3 : 0;

   }

   return size;

 }

 int MachEpilogNode::reloc() const {

   return 0; // a large enough number

 }

 const Pipeline * MachEpilogNode::pipeline() const {

   return MachNode::pipeline_class();

 }

 int MachEpilogNode::safepoint_offset() const { return 0; }

 //=============================================================================

 enum RC { rc_bad, rc_int, rc_float, rc_xmm, rc_stack };

 static enum RC rc_class( OptoReg::Name reg ) {

   if( !OptoReg::is_valid(reg)  ) return rc_bad;

   if (OptoReg::is_stack(reg)) return rc_stack;

   VMReg r = OptoReg::as_VMReg(reg);

   if (r->is_Register()) return rc_int;

   if (r->is_FloatRegister()) {

     assert(UseSSE < 2, "shouldn't be used in SSE2+ mode");

     return rc_float;

   }

   assert(r->is_XMMRegister(), "must be");

   return rc_xmm;

 }

 static int impl_helper( CodeBuffer *cbuf, bool do_size, bool is_load, int offset, int reg,

                         int opcode, const char *op_str, int size, outputStream* st ) {

   if( cbuf ) {

     emit_opcode  (*cbuf, opcode );

     encode_RegMem(*cbuf, Matcher::_regEncode[reg], ESP_enc, 0x4, 0, offset, false);

 #ifndef PRODUCT

   } else if( !do_size ) {

     if( size != 0 ) st->print("\n\t");

     if( opcode == 0x8B || opcode == 0x89 ) { // MOV

       if( is_load ) st->print("%s   %s,[ESP + #%d]",op_str,Matcher::regName[reg],offset);

       else          st->print("%s   [ESP + #%d],%s",op_str,offset,Matcher::regName[reg]);

     } else { // FLD, FST, PUSH, POP

       st->print("%s [ESP + #%d]",op_str,offset);

     }

 #endif

   }

   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);

   return size+3+offset_size;

 }

 // Helper for XMM registers.  Extra opcode bits, limited syntax.

 static int impl_x_helper( CodeBuffer *cbuf, bool do_size, bool is_load,

                          int offset, int reg_lo, int reg_hi, int size, outputStream* st ) {

   if( cbuf ) {

     if( reg_lo+1 == reg_hi ) { // double move?

       if( is_load && !UseXmmLoadAndClearUpper )

         emit_opcode(*cbuf, 0x66 ); // use 'movlpd' for load

       else

         emit_opcode(*cbuf, 0xF2 ); // use 'movsd' otherwise

     } else {

       emit_opcode(*cbuf, 0xF3 );

     }

     emit_opcode(*cbuf, 0x0F );

     if( reg_lo+1 == reg_hi && is_load && !UseXmmLoadAndClearUpper )

       emit_opcode(*cbuf, 0x12 );   // use 'movlpd' for load

     else

       emit_opcode(*cbuf, is_load ? 0x10 : 0x11 );

     encode_RegMem(*cbuf, Matcher::_regEncode[reg_lo], ESP_enc, 0x4, 0, offset, false);

 #ifndef PRODUCT

   } else if( !do_size ) {

     if( size != 0 ) st->print("\n\t");

     if( reg_lo+1 == reg_hi ) { // double move?

       if( is_load ) st->print("%s %s,[ESP + #%d]",

                                UseXmmLoadAndClearUpper ? "MOVSD " : "MOVLPD",

                                Matcher::regName[reg_lo], offset);

       else          st->print("MOVSD  [ESP + #%d],%s",

                                offset, Matcher::regName[reg_lo]);

     } else {

       if( is_load ) st->print("MOVSS  %s,[ESP + #%d]",

                                Matcher::regName[reg_lo], offset);

       else          st->print("MOVSS  [ESP + #%d],%s",

                                offset, Matcher::regName[reg_lo]);

     }

 #endif

   }

   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);

   return size+5+offset_size;

 }

 static int impl_movx_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,

                             int src_hi, int dst_hi, int size, outputStream* st ) {

   if( UseXmmRegToRegMoveAll ) {//Use movaps,movapd to move between xmm registers

     if( cbuf ) {

       if( (src_lo+1 == src_hi && dst_lo+1 == dst_hi) ) {

         emit_opcode(*cbuf, 0x66 );

       }

       emit_opcode(*cbuf, 0x0F );

       emit_opcode(*cbuf, 0x28 );

       emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst_lo], Matcher::_regEncode[src_lo] );

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?

         st->print("MOVAPD %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);

       } else {

         st->print("MOVAPS %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);

       }

 #endif

     }

     return size + ((src_lo+1 == src_hi && dst_lo+1 == dst_hi) ? 4 : 3);

   } else {

     if( cbuf ) {

       emit_opcode(*cbuf, (src_lo+1 == src_hi && dst_lo+1 == dst_hi) ? 0xF2 : 0xF3 );

       emit_opcode(*cbuf, 0x0F );

       emit_opcode(*cbuf, 0x10 );

       emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst_lo], Matcher::_regEncode[src_lo] );

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?

         st->print("MOVSD  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);

       } else {

         st->print("MOVSS  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);

       }

 #endif

     }

     return size+4;

   }

 }

 static int impl_movgpr2x_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,

                             int src_hi, int dst_hi, int size, outputStream* st ) {

   // 32-bit

   if (cbuf) {

     emit_opcode(*cbuf, 0x66);

     emit_opcode(*cbuf, 0x0F);

     emit_opcode(*cbuf, 0x6E);

     emit_rm(*cbuf, 0x3, Matcher::_regEncode[dst_lo] & 7, Matcher::_regEncode[src_lo] & 7);

 #ifndef PRODUCT

   } else if (!do_size) {

     st->print("movdl   %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);

 #endif

   }

   return 4;

 }

 static int impl_movx2gpr_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,

                                  int src_hi, int dst_hi, int size, outputStream* st ) {

   // 32-bit

   if (cbuf) {

     emit_opcode(*cbuf, 0x66);

     emit_opcode(*cbuf, 0x0F);

     emit_opcode(*cbuf, 0x7E);

     emit_rm(*cbuf, 0x3, Matcher::_regEncode[src_lo] & 7, Matcher::_regEncode[dst_lo] & 7);

 #ifndef PRODUCT

   } else if (!do_size) {

     st->print("movdl   %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);

 #endif

   }

   return 4;

 }

 static int impl_mov_helper( CodeBuffer *cbuf, bool do_size, int src, int dst, int size, outputStream* st ) {

   if( cbuf ) {

     emit_opcode(*cbuf, 0x8B );

     emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst], Matcher::_regEncode[src] );

 #ifndef PRODUCT

   } else if( !do_size ) {

     if( size != 0 ) st->print("\n\t");

     st->print("MOV    %s,%s",Matcher::regName[dst],Matcher::regName[src]);

 #endif

   }

   return size+2;

 }

 static int impl_fp_store_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int src_hi, int dst_lo, int dst_hi,

                                  int offset, int size, outputStream* st ) {

   if( src_lo != FPR1L_num ) {      // Move value to top of FP stack, if not already there

     if( cbuf ) {

       emit_opcode( *cbuf, 0xD9 );  // FLD (i.e., push it)

       emit_d8( *cbuf, 0xC0-1+Matcher::_regEncode[src_lo] );

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       st->print("FLD    %s",Matcher::regName[src_lo]);

 #endif

     }

     size += 2;

   }

   int st_op = (src_lo != FPR1L_num) ? EBX_num /*store & pop*/ : EDX_num /*store no pop*/;

   const char *op_str;

   int op;

   if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double store?

     op_str = (src_lo != FPR1L_num) ? "FSTP_D" : "FST_D ";

     op = 0xDD;

   } else {                   // 32-bit store

     op_str = (src_lo != FPR1L_num) ? "FSTP_S" : "FST_S ";

     op = 0xD9;

     assert( !OptoReg::is_valid(src_hi) && !OptoReg::is_valid(dst_hi), "no non-adjacent float-stores" );

   }

   return impl_helper(cbuf,do_size,false,offset,st_op,op,op_str,size, st);

 }

 uint MachSpillCopyNode::implementation( CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream* st ) const {

   // Get registers to move

   OptoReg::Name src_second = ra_->get_reg_second(in(1));

   OptoReg::Name src_first = ra_->get_reg_first(in(1));

   OptoReg::Name dst_second = ra_->get_reg_second(this );

   OptoReg::Name dst_first = ra_->get_reg_first(this );

   enum RC src_second_rc = rc_class(src_second);

   enum RC src_first_rc = rc_class(src_first);

   enum RC dst_second_rc = rc_class(dst_second);

   enum RC dst_first_rc = rc_class(dst_first);

   assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );

   // Generate spill code!

   int size = 0;

   if( src_first == dst_first && src_second == dst_second )

     return size;            // Self copy, no move

   // --------------------------------------

   // Check for mem-mem move.  push/pop to move.

   if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {

     if( src_second == dst_first ) { // overlapping stack copy ranges

       assert( src_second_rc == rc_stack && dst_second_rc == rc_stack, "we only expect a stk-stk copy here" );

       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);

       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);

       src_second_rc = dst_second_rc = rc_bad;  // flag as already moved the second bits

     }

     // move low bits

     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),ESI_num,0xFF,"PUSH  ",size, st);

     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),EAX_num,0x8F,"POP   ",size, st);

     if( src_second_rc == rc_stack && dst_second_rc == rc_stack ) { // mov second bits

       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);

       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);

     }

     return size;

   }

   // --------------------------------------

   // Check for integer reg-reg copy

   if( src_first_rc == rc_int && dst_first_rc == rc_int )

     size = impl_mov_helper(cbuf,do_size,src_first,dst_first,size, st);

   // Check for integer store

   if( src_first_rc == rc_int && dst_first_rc == rc_stack )

     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first,0x89,"MOV ",size, st);

   // Check for integer load

   if( dst_first_rc == rc_int && src_first_rc == rc_stack )

     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first,0x8B,"MOV ",size, st);

   // Check for integer reg-xmm reg copy

   if( src_first_rc == rc_int && dst_first_rc == rc_xmm ) {

     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),

             "no 64 bit integer-float reg moves" );

     return impl_movgpr2x_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);

   }

   // --------------------------------------

   // Check for float reg-reg copy

   if( src_first_rc == rc_float && dst_first_rc == rc_float ) {

     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||

             (src_first+1 == src_second && dst_first+1 == dst_second), "no non-adjacent float-moves" );

     if( cbuf ) {

       // Note the mucking with the register encode to compensate for the 0/1

       // indexing issue mentioned in a comment in the reg_def sections

       // for FPR registers many lines above here.

       if( src_first != FPR1L_num ) {

         emit_opcode  (*cbuf, 0xD9 );           // FLD    ST(i)

         emit_d8      (*cbuf, 0xC0+Matcher::_regEncode[src_first]-1 );

         emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)

         emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );

      } else {

         emit_opcode  (*cbuf, 0xDD );           // FST    ST(i)

         emit_d8      (*cbuf, 0xD0+Matcher::_regEncode[dst_first]-1 );

      }

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       if( src_first != FPR1L_num ) st->print("FLD    %s\n\tFSTP   %s",Matcher::regName[src_first],Matcher::regName[dst_first]);

       else                      st->print(             "FST    %s",                            Matcher::regName[dst_first]);

 #endif

     }

     return size + ((src_first != FPR1L_num) ? 2+2 : 2);

   }

   // Check for float store

   if( src_first_rc == rc_float && dst_first_rc == rc_stack ) {

     return impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,ra_->reg2offset(dst_first),size, st);

   }

   // Check for float load

   if( dst_first_rc == rc_float && src_first_rc == rc_stack ) {

     int offset = ra_->reg2offset(src_first);

     const char *op_str;

     int op;

     if( src_first+1 == src_second && dst_first+1 == dst_second ) { // double load?

       op_str = "FLD_D";

       op = 0xDD;

     } else {                   // 32-bit load

       op_str = "FLD_S";

       op = 0xD9;

       assert( src_second_rc == rc_bad && dst_second_rc == rc_bad, "no non-adjacent float-loads" );

     }

     if( cbuf ) {

       emit_opcode  (*cbuf, op );

       encode_RegMem(*cbuf, 0x0, ESP_enc, 0x4, 0, offset, false);

       emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)

       emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       st->print("%s  ST,[ESP + #%d]\n\tFSTP   %s",op_str, offset,Matcher::regName[dst_first]);

 #endif

     }

     int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);

     return size + 3+offset_size+2;

   }

   // Check for xmm reg-reg copy

   if( src_first_rc == rc_xmm && dst_first_rc == rc_xmm ) {

     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||

             (src_first+1 == src_second && dst_first+1 == dst_second),

             "no non-adjacent float-moves" );

     return impl_movx_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);

   }

   // Check for xmm reg-integer reg copy

   if( src_first_rc == rc_xmm && dst_first_rc == rc_int ) {

     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),

             "no 64 bit float-integer reg moves" );

     return impl_movx2gpr_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);

   }

   // Check for xmm store

   if( src_first_rc == rc_xmm && dst_first_rc == rc_stack ) {

     return impl_x_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first, src_second, size, st);

   }

   // Check for float xmm load

   if( dst_first_rc == rc_xmm && src_first_rc == rc_stack ) {

     return impl_x_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first, dst_second, size, st);

   }

   // Copy from float reg to xmm reg

   if( dst_first_rc == rc_xmm && src_first_rc == rc_float ) {

     // copy to the top of stack from floating point reg

     // and use LEA to preserve flags

     if( cbuf ) {

       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP-8]

       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);

       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);

       emit_d8(*cbuf,0xF8);

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       st->print("LEA    ESP,[ESP-8]");

 #endif

     }

     size += 4;

     size = impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,0,size, st);

     // Copy from the temp memory to the xmm reg.

     size = impl_x_helper(cbuf,do_size,true ,0,dst_first, dst_second, size, st);

     if( cbuf ) {

       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP+8]

       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);

       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);

       emit_d8(*cbuf,0x08);

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       st->print("LEA    ESP,[ESP+8]");

 #endif

     }

     size += 4;

     return size;

   }

   assert( size > 0, "missed a case" );

   // --------------------------------------------------------------------

   // Check for second bits still needing moving.

   if( src_second == dst_second )

     return size;               // Self copy; no move

   assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );

   // Check for second word int-int move

   if( src_second_rc == rc_int && dst_second_rc == rc_int )

     return impl_mov_helper(cbuf,do_size,src_second,dst_second,size, st);

   // Check for second word integer store

   if( src_second_rc == rc_int && dst_second_rc == rc_stack )

     return impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),src_second,0x89,"MOV ",size, st);

   // Check for second word integer load

   if( dst_second_rc == rc_int && src_second_rc == rc_stack )

     return impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),dst_second,0x8B,"MOV ",size, st);

   Unimplemented();

 }

 #ifndef PRODUCT

 void MachSpillCopyNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {

   implementation( NULL, ra_, false, st );

 }

 #endif

 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

   implementation( &cbuf, ra_, false, NULL );

 }

 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {

   return implementation( NULL, ra_, true, NULL );

 }

 //=============================================================================

 #ifndef PRODUCT

 void MachNopNode::format( PhaseRegAlloc *, outputStream* st ) const {

   st->print("NOP \t# %d bytes pad for loops and calls", _count);

 }

 #endif

 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc * ) const {

   MacroAssembler _masm(&cbuf);

   __ nop(_count);

 }

 uint MachNopNode::size(PhaseRegAlloc *) const {

   return _count;

 }

 //=============================================================================

 #ifndef PRODUCT

 void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {

   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());

   int reg = ra_->get_reg_first(this);

   st->print("LEA    %s,[ESP + #%d]",Matcher::regName[reg],offset);

 }

 #endif

 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());

   int reg = ra_->get_encode(this);

   if( offset >= 128 ) {

     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]

     emit_rm(cbuf, 0x2, reg, 0x04);

     emit_rm(cbuf, 0x0, 0x04, ESP_enc);

     emit_d32(cbuf, offset);

   }

   else {

     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]

     emit_rm(cbuf, 0x1, reg, 0x04);

     emit_rm(cbuf, 0x0, 0x04, ESP_enc);

     emit_d8(cbuf, offset);

   }

 }

 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {

   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());

   if( offset >= 128 ) {

     return 7;

   }

   else {

     return 4;

   }

 }

 //=============================================================================

 // emit call stub, compiled java to interpreter

 void emit_java_to_interp(CodeBuffer &cbuf ) {

   // Stub is fixed up when the corresponding call is converted from calling

   // compiled code to calling interpreted code.

   // mov rbx,0

   // jmp -1

   address mark = cbuf.inst_mark();  // get mark within main instrs section

   // Note that the code buffer's inst_mark is always relative to insts.

   // That's why we must use the macroassembler to generate a stub.

   MacroAssembler _masm(&cbuf);

   address base =

   __ start_a_stub(Compile::MAX_stubs_size);

   if (base == NULL)  return;  // CodeBuffer::expand failed

   // static stub relocation stores the instruction address of the call

   __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM32);

   // static stub relocation also tags the methodOop in the code-stream.

   __ movoop(rbx, (jobject)NULL);  // method is zapped till fixup time

   // This is recognized as unresolved by relocs/nativeInst/ic code

   __ jump(RuntimeAddress(__ pc()));

   __ end_a_stub();

   // Update current stubs pointer and restore code_end.

 }

 // size of call stub, compiled java to interpretor

 uint size_java_to_interp() {

   return 10;  // movl; jmp

 }

 // relocation entries for call stub, compiled java to interpretor

 uint reloc_java_to_interp() {

   return 4;  // 3 in emit_java_to_interp + 1 in Java_Static_Call

 }

 //=============================================================================

 #ifndef PRODUCT

 void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {

   st->print_cr(  "CMP    EAX,[ECX+4]\t# Inline cache check");

   st->print_cr("\tJNE    SharedRuntime::handle_ic_miss_stub");

   st->print_cr("\tNOP");

   st->print_cr("\tNOP");

   if( !OptoBreakpoint )

     st->print_cr("\tNOP");

 }

 #endif

 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

   MacroAssembler masm(&cbuf);

 #ifdef ASSERT

   uint code_size = cbuf.code_size();

 #endif

   masm.cmpptr(rax, Address(rcx, oopDesc::klass_offset_in_bytes()));

   masm.jump_cc(Assembler::notEqual,

                RuntimeAddress(SharedRuntime::get_ic_miss_stub()));

   /* WARNING these NOPs are critical so that verified entry point is properly

      aligned for patching by NativeJump::patch_verified_entry() */

   int nops_cnt = 2;

   if( !OptoBreakpoint ) // Leave space for int3

      nops_cnt += 1;

   masm.nop(nops_cnt);

   assert(cbuf.code_size() - code_size == size(ra_), "checking code size of inline cache node");

 }

 uint MachUEPNode::size(PhaseRegAlloc *ra_) const {

   return OptoBreakpoint ? 11 : 12;

 }

 //=============================================================================

 uint size_exception_handler() {

   // NativeCall instruction size is the same as NativeJump.

   // exception handler starts out as jump and can be patched to

   // a call be deoptimization.  (4932387)

   // Note that this value is also credited (in output.cpp) to

   // the size of the code section.

   return NativeJump::instruction_size;

 }

 // Emit exception handler code.  Stuff framesize into a register

 // and call a VM stub routine.

 int emit_exception_handler(CodeBuffer& cbuf) {

   // Note that the code buffer's inst_mark is always relative to insts.

   // That's why we must use the macroassembler to generate a handler.

   MacroAssembler _masm(&cbuf);

   address base =

   __ start_a_stub(size_exception_handler());

   if (base == NULL)  return 0;  // CodeBuffer::expand failed

   int offset = __ offset();

   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->instructions_begin()));

   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");

   __ end_a_stub();

   return offset;

 }

 uint size_deopt_handler() {

   // NativeCall instruction size is the same as NativeJump.

   // exception handler starts out as jump and can be patched to

   // a call be deoptimization.  (4932387)

   // Note that this value is also credited (in output.cpp) to

   // the size of the code section.

   return 5 + NativeJump::instruction_size; // pushl(); jmp;

 }

 // Emit deopt handler code.

 int emit_deopt_handler(CodeBuffer& cbuf) {

   // Note that the code buffer's inst_mark is always relative to insts.

   // That's why we must use the macroassembler to generate a handler.

   MacroAssembler _masm(&cbuf);

   address base =

   __ start_a_stub(size_exception_handler());

   if (base == NULL)  return 0;  // CodeBuffer::expand failed

   int offset = __ offset();

   InternalAddress here(__ pc());

   __ pushptr(here.addr());

   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));

   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");

   __ end_a_stub();

   return offset;

 }

 static void emit_double_constant(CodeBuffer& cbuf, double x) {

   int mark = cbuf.insts()->mark_off();

   MacroAssembler _masm(&cbuf);

   address double_address = __ double_constant(x);

   cbuf.insts()->set_mark_off(mark);  // preserve mark across masm shift

   emit_d32_reloc(cbuf,

                  (int)double_address,

                  internal_word_Relocation::spec(double_address),

                  RELOC_DISP32);

 }

 static void emit_float_constant(CodeBuffer& cbuf, float x) {

   int mark = cbuf.insts()->mark_off();

   MacroAssembler _masm(&cbuf);

   address float_address = __ float_constant(x);

   cbuf.insts()->set_mark_off(mark);  // preserve mark across masm shift

   emit_d32_reloc(cbuf,

                  (int)float_address,

                  internal_word_Relocation::spec(float_address),

                  RELOC_DISP32);

 }

 const bool Matcher::match_rule_supported(int opcode) {

   if (!has_match_rule(opcode))

     return false;

   return true;  // Per default match rules are supported.

 }

 int Matcher::regnum_to_fpu_offset(int regnum) {

   return regnum - 32; // The FP registers are in the second chunk

 }

 bool is_positive_zero_float(jfloat f) {

   return jint_cast(f) == jint_cast(0.0F);

 }

 bool is_positive_one_float(jfloat f) {

   return jint_cast(f) == jint_cast(1.0F);

 }

 bool is_positive_zero_double(jdouble d) {

   return jlong_cast(d) == jlong_cast(0.0);

 }

 bool is_positive_one_double(jdouble d) {

   return jlong_cast(d) == jlong_cast(1.0);

 }

 // This is UltraSparc specific, true just means we have fast l2f conversion

 const bool Matcher::convL2FSupported(void) {

   return true;

 }

 // Vector width in bytes

 const uint Matcher::vector_width_in_bytes(void) {

   return UseSSE >= 2 ? 8 : 0;

 }

 // Vector ideal reg

 const uint Matcher::vector_ideal_reg(void) {

   return Op_RegD;

 }

 // Is this branch offset short enough that a short branch can be used?

 //

 // NOTE: If the platform does not provide any short branch variants, then

 //       this method should return false for offset 0.

 bool Matcher::is_short_branch_offset(int rule, int offset) {

   // the short version of jmpConUCF2 contains multiple branches,

   // making the reach slightly less

   if (rule == jmpConUCF2_rule)

     return (-126 <= offset && offset <= 125);

   return (-128 <= offset && offset <= 127);

 }

 const bool Matcher::isSimpleConstant64(jlong value) {

   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.

   return false;

 }

 // The ecx parameter to rep stos for the ClearArray node is in dwords.

 const bool Matcher::init_array_count_is_in_bytes = false;

 // Threshold size for cleararray.

 const int Matcher::init_array_short_size = 8 * BytesPerLong;

 // Should the Matcher clone shifts on addressing modes, expecting them to

 // be subsumed into complex addressing expressions or compute them into

 // registers?  True for Intel but false for most RISCs

 const bool Matcher::clone_shift_expressions = true;

 bool Matcher::narrow_oop_use_complex_address() {

   ShouldNotCallThis();

   return true;

 }

 // Is it better to copy float constants, or load them directly from memory?

 // Intel can load a float constant from a direct address, requiring no

 // extra registers.  Most RISCs will have to materialize an address into a

 // register first, so they would do better to copy the constant from stack.

 const bool Matcher::rematerialize_float_constants = true;

 // If CPU can load and store mis-aligned doubles directly then no fixup is

 // needed.  Else we split the double into 2 integer pieces and move it

 // piece-by-piece.  Only happens when passing doubles into C code as the

 // Java calling convention forces doubles to be aligned.

 const bool Matcher::misaligned_doubles_ok = true;

 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {

   // Get the memory operand from the node

   uint numopnds = node->num_opnds();        // Virtual call for number of operands

   uint skipped  = node->oper_input_base();  // Sum of leaves skipped so far

   assert( idx >= skipped, "idx too low in pd_implicit_null_fixup" );

   uint opcnt     = 1;                 // First operand

   uint num_edges = node->_opnds[1]->num_edges(); // leaves for first operand

   while( idx >= skipped+num_edges ) {

     skipped += num_edges;

     opcnt++;                          // Bump operand count

     assert( opcnt < numopnds, "Accessing non-existent operand" );

     num_edges = node->_opnds[opcnt]->num_edges(); // leaves for next operand

   }

   MachOper *memory = node->_opnds[opcnt];

   MachOper *new_memory = NULL;

   switch (memory->opcode()) {

   case DIRECT:

   case INDOFFSET32X:

     // No transformation necessary.

     return;

   case INDIRECT:

     new_memory = new (C) indirect_win95_safeOper( );

     break;

   case INDOFFSET8:

     new_memory = new (C) indOffset8_win95_safeOper(memory->disp(NULL, NULL, 0));

     break;

   case INDOFFSET32:

     new_memory = new (C) indOffset32_win95_safeOper(memory->disp(NULL, NULL, 0));

     break;

   case INDINDEXOFFSET:

     new_memory = new (C) indIndexOffset_win95_safeOper(memory->disp(NULL, NULL, 0));

     break;

   case INDINDEXSCALE:

     new_memory = new (C) indIndexScale_win95_safeOper(memory->scale());

     break;

   case INDINDEXSCALEOFFSET:

     new_memory = new (C) indIndexScaleOffset_win95_safeOper(memory->scale(), memory->disp(NULL, NULL, 0));

     break;

   case LOAD_LONG_INDIRECT:

   case LOAD_LONG_INDOFFSET32:

     // Does not use EBP as address register, use { EDX, EBX, EDI, ESI}

     return;

   default:

     assert(false, "unexpected memory operand in pd_implicit_null_fixup()");

     return;

   }

   node->_opnds[opcnt] = new_memory;

 }

 // Advertise here if the CPU requires explicit rounding operations

 // to implement the UseStrictFP mode.

 const bool Matcher::strict_fp_requires_explicit_rounding = true;

 // Are floats conerted to double when stored to stack during deoptimization?

 // On x32 it is stored with convertion only when FPU is used for floats.

 bool Matcher::float_in_double() { return (UseSSE == 0); }

 // Do ints take an entire long register or just half?

 const bool Matcher::int_in_long = false;

 // Return whether or not this register is ever used as an argument.  This

 // function is used on startup to build the trampoline stubs in generateOptoStub.

 // Registers not mentioned will be killed by the VM call in the trampoline, and

 // arguments in those registers not be available to the callee.

 bool Matcher::can_be_java_arg( int reg ) {

   if(  reg == ECX_num   || reg == EDX_num   ) return true;

   if( (reg == XMM0a_num || reg == XMM1a_num) && UseSSE>=1 ) return true;

   if( (reg == XMM0b_num || reg == XMM1b_num) && UseSSE>=2 ) return true;

   return false;

 }

 bool Matcher::is_spillable_arg( int reg ) {

   return can_be_java_arg(reg);

 }

 // Register for DIVI projection of divmodI

 RegMask Matcher::divI_proj_mask() {

   return EAX_REG_mask;

 }

 // Register for MODI projection of divmodI

 RegMask Matcher::modI_proj_mask() {

   return EDX_REG_mask;

 }

 // Register for DIVL projection of divmodL

 RegMask Matcher::divL_proj_mask() {

   ShouldNotReachHere();

   return RegMask();

 }

 // Register for MODL projection of divmodL

 RegMask Matcher::modL_proj_mask() {

   ShouldNotReachHere();

   return RegMask();

 }

 const RegMask Matcher::method_handle_invoke_SP_save_mask() {

   return EBP_REG_mask;

 }

 // Returns true if the high 32 bits of the value is known to be zero.

 bool is_operand_hi32_zero(Node* n) {

   int opc = n->Opcode();

   if (opc == Op_LoadUI2L) {

     return true;

   }

   if (opc == Op_AndL) {

     Node* o2 = n->in(2);

     if (o2->is_Con() && (o2->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {

       return true;

     }

   }

   return false;

 }

 %}

 //----------ENCODING BLOCK-----------------------------------------------------

 // This block specifies the encoding classes used by the compiler to output

 // byte streams.  Encoding classes generate functions which are called by

 // Machine Instruction Nodes in order to generate the bit encoding of the

 // instruction.  Operands specify their base encoding interface with the

 // interface keyword.  There are currently supported four interfaces,

 // REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER.  REG_INTER causes an

 // operand to generate a function which returns its register number when

 // queried.   CONST_INTER causes an operand to generate a function which

 // returns the value of the constant when queried.  MEMORY_INTER causes an

 // operand to generate four functions which return the Base Register, the

 // Index Register, the Scale Value, and the Offset Value of the operand when

 // queried.  COND_INTER causes an operand to generate six functions which

 // return the encoding code (ie - encoding bits for the instruction)

 // associated with each basic boolean condition for a conditional instruction.

 // Instructions specify two basic values for encoding.  They use the

 // ins_encode keyword to specify their encoding class (which must be one of

 // the class names specified in the encoding block), and they use the

 // opcode keyword to specify, in order, their primary, secondary, and

 // tertiary opcode.  Only the opcode sections which a particular instruction

 // needs for encoding need to be specified.

 encode %{

   // Build emit functions for each basic byte or larger field in the intel

   // encoding scheme (opcode, rm, sib, immediate), and call them from C++

   // code in the enc_class source block.  Emit functions will live in the

   // main source block for now.  In future, we can generalize this by

   // adding a syntax that specifies the sizes of fields in an order,

   // so that the adlc can build the emit functions automagically

   // Emit primary opcode

   enc_class OpcP %{

     emit_opcode(cbuf, $primary);

   %}

   // Emit secondary opcode

   enc_class OpcS %{

     emit_opcode(cbuf, $secondary);

   %}

   // Emit opcode directly

   enc_class Opcode(immI d8) %{

     emit_opcode(cbuf, $d8$$constant);

   %}

   enc_class SizePrefix %{

     emit_opcode(cbuf,0x66);

   %}

   enc_class RegReg (eRegI dst, eRegI src) %{    // RegReg(Many)

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class OpcRegReg (immI opcode, eRegI dst, eRegI src) %{    // OpcRegReg(Many)

     emit_opcode(cbuf,$opcode$$constant);

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class mov_r32_imm0( eRegI dst ) %{

     emit_opcode( cbuf, 0xB8 + $dst$$reg ); // 0xB8+ rd   -- MOV r32  ,imm32

     emit_d32   ( cbuf, 0x0  );             //                         imm32==0x0

   %}

   enc_class cdq_enc %{

     // Full implementation of Java idiv and irem; checks for

     // special case as described in JVM spec., p.243 & p.271.

     //

     //         normal case                           special case

     //

     // input : rax,: dividend                         min_int

     //         reg: divisor                          -1

     //

     // output: rax,: quotient  (= rax, idiv reg)       min_int

     //         rdx: remainder (= rax, irem reg)       0

     //

     //  Code sequnce:

     //

     //  81 F8 00 00 00 80    cmp         rax,80000000h

     //  0F 85 0B 00 00 00    jne         normal_case

     //  33 D2                xor         rdx,edx

     //  83 F9 FF             cmp         rcx,0FFh

     //  0F 84 03 00 00 00    je          done

     //                  normal_case:

     //  99                   cdq

     //  F7 F9                idiv        rax,ecx

     //                  done:

     //

     emit_opcode(cbuf,0x81); emit_d8(cbuf,0xF8);

     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);

     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x80);                     // cmp rax,80000000h

     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x85);

     emit_opcode(cbuf,0x0B); emit_d8(cbuf,0x00);

     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // jne normal_case

     emit_opcode(cbuf,0x33); emit_d8(cbuf,0xD2);                     // xor rdx,edx

     emit_opcode(cbuf,0x83); emit_d8(cbuf,0xF9); emit_d8(cbuf,0xFF); // cmp rcx,0FFh

     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x84);

     emit_opcode(cbuf,0x03); emit_d8(cbuf,0x00);

     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // je done

     // normal_case:

     emit_opcode(cbuf,0x99);                                         // cdq

     // idiv (note: must be emitted by the user of this rule)

     // normal:

   %}

   // Dense encoding for older common ops

   enc_class Opc_plus(immI opcode, eRegI reg) %{

     emit_opcode(cbuf, $opcode$$constant + $reg$$reg);

   %}

   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension

   enc_class OpcSE (immI imm) %{ // Emit primary opcode and set sign-extend bit

     // Check for 8-bit immediate, and set sign extend bit in opcode

     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {

       emit_opcode(cbuf, $primary | 0x02);

     }

     else {                          // If 32-bit immediate

       emit_opcode(cbuf, $primary);

     }

   %}

   enc_class OpcSErm (eRegI dst, immI imm) %{    // OpcSEr/m

     // Emit primary opcode and set sign-extend bit

     // Check for 8-bit immediate, and set sign extend bit in opcode

     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {

       emit_opcode(cbuf, $primary | 0x02);    }

     else {                          // If 32-bit immediate

       emit_opcode(cbuf, $primary);

     }

     // Emit r/m byte with secondary opcode, after primary opcode.

     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);

   %}

   enc_class Con8or32 (immI imm) %{    // Con8or32(storeImmI), 8 or 32 bits

     // Check for 8-bit immediate, and set sign extend bit in opcode

     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {

       $$$emit8$imm$$constant;

     }

     else {                          // If 32-bit immediate

       // Output immediate

       $$$emit32$imm$$constant;

     }

   %}

   enc_class Long_OpcSErm_Lo(eRegL dst, immL imm) %{

     // Emit primary opcode and set sign-extend bit

     // Check for 8-bit immediate, and set sign extend bit in opcode

     int con = (int)$imm$$constant; // Throw away top bits

     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);

     // Emit r/m byte with secondary opcode, after primary opcode.

     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);

     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);

     else                               emit_d32(cbuf,con);

   %}

   enc_class Long_OpcSErm_Hi(eRegL dst, immL imm) %{

     // Emit primary opcode and set sign-extend bit

     // Check for 8-bit immediate, and set sign extend bit in opcode

     int con = (int)($imm$$constant >> 32); // Throw away bottom bits

     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);

     // Emit r/m byte with tertiary opcode, after primary opcode.

     emit_rm(cbuf, 0x3, $tertiary, HIGH_FROM_LOW($dst$$reg));

     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);

     else                               emit_d32(cbuf,con);

   %}

   enc_class Lbl (label labl) %{ // JMP, CALL

     Label *l = $labl$$label;

     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.code_size()+4)) : 0);

   %}

   enc_class LblShort (label labl) %{ // JMP, CALL

     Label *l = $labl$$label;

     int disp = l ? (l->loc_pos() - (cbuf.code_size()+1)) : 0;

     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");

     emit_d8(cbuf, disp);

   %}

   enc_class OpcSReg (eRegI dst) %{    // BSWAP

     emit_cc(cbuf, $secondary, $dst$$reg );

   %}

   enc_class bswap_long_bytes(eRegL dst) %{ // BSWAP

     int destlo = $dst$$reg;

     int desthi = HIGH_FROM_LOW(destlo);

     // bswap lo

     emit_opcode(cbuf, 0x0F);

     emit_cc(cbuf, 0xC8, destlo);

     // bswap hi

     emit_opcode(cbuf, 0x0F);

     emit_cc(cbuf, 0xC8, desthi);

     // xchg lo and hi

     emit_opcode(cbuf, 0x87);

     emit_rm(cbuf, 0x3, destlo, desthi);

   %}

   enc_class RegOpc (eRegI div) %{    // IDIV, IMOD, JMP indirect, ...

     emit_rm(cbuf, 0x3, $secondary, $div$$reg );

   %}

   enc_class Jcc (cmpOp cop, label labl) %{    // JCC

     Label *l = $labl$$label;

     $$$emit8$primary;

     emit_cc(cbuf, $secondary, $cop$$cmpcode);

     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.code_size()+4)) : 0);

   %}

   enc_class JccShort (cmpOp cop, label labl) %{    // JCC

     Label *l = $labl$$label;

     emit_cc(cbuf, $primary, $cop$$cmpcode);

     int disp = l ? (l->loc_pos() - (cbuf.code_size()+1)) : 0;

     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");

     emit_d8(cbuf, disp);

   %}

   enc_class enc_cmov(cmpOp cop ) %{ // CMOV

     $$$emit8$primary;

     emit_cc(cbuf, $secondary, $cop$$cmpcode);

   %}

   enc_class enc_cmov_d(cmpOp cop, regD src ) %{ // CMOV

     int op = 0xDA00 + $cop$$cmpcode + ($src$$reg-1);

     emit_d8(cbuf, op >> 8 );

     emit_d8(cbuf, op & 255);

   %}

   // emulate a CMOV with a conditional branch around a MOV

   enc_class enc_cmov_branch( cmpOp cop, immI brOffs ) %{ // CMOV

     // Invert sense of branch from sense of CMOV

     emit_cc( cbuf, 0x70, ($cop$$cmpcode^1) );

     emit_d8( cbuf, $brOffs$$constant );

   %}

   enc_class enc_PartialSubtypeCheck( ) %{

     Register Redi = as_Register(EDI_enc); // result register

     Register Reax = as_Register(EAX_enc); // super class

     Register Recx = as_Register(ECX_enc); // killed

     Register Resi = as_Register(ESI_enc); // sub class

     Label miss;

     MacroAssembler _masm(&cbuf);

     __ check_klass_subtype_slow_path(Resi, Reax, Recx, Redi,

                                      NULL, &miss,

                                      /*set_cond_codes:*/ true);

     if ($primary) {

       __ xorptr(Redi, Redi);

     }

     __ bind(miss);

   %}

   enc_class FFree_Float_Stack_All %{    // Free_Float_Stack_All

     MacroAssembler masm(&cbuf);

     int start = masm.offset();

     if (UseSSE >= 2) {

       if (VerifyFPU) {

         masm.verify_FPU(0, "must be empty in SSE2+ mode");

       }

     } else {

       // External c_calling_convention expects the FPU stack to be 'clean'.

       // Compiled code leaves it dirty.  Do cleanup now.

       masm.empty_FPU_stack();

     }

     if (sizeof_FFree_Float_Stack_All == -1) {

       sizeof_FFree_Float_Stack_All = masm.offset() - start;

     } else {

       assert(masm.offset() - start == sizeof_FFree_Float_Stack_All, "wrong size");

     }

   %}

   enc_class Verify_FPU_For_Leaf %{

     if( VerifyFPU ) {

       MacroAssembler masm(&cbuf);

       masm.verify_FPU( -3, "Returning from Runtime Leaf call");

     }

   %}

   enc_class Java_To_Runtime (method meth) %{    // CALL Java_To_Runtime, Java_To_Runtime_Leaf

     // This is the instruction starting address for relocation info.

     cbuf.set_inst_mark();

     $$$emit8$primary;

     // CALL directly to the runtime

     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),

                 runtime_call_Relocation::spec(), RELOC_IMM32 );

     if (UseSSE >= 2) {

       MacroAssembler _masm(&cbuf);

       BasicType rt = tf()->return_type();

       if ((rt == T_FLOAT || rt == T_DOUBLE) && !return_value_is_used()) {

         // A C runtime call where the return value is unused.  In SSE2+

         // mode the result needs to be removed from the FPU stack.  It's

         // likely that this function call could be removed by the

         // optimizer if the C function is a pure function.

         __ ffree(0);

       } else if (rt == T_FLOAT) {

         __ lea(rsp, Address(rsp, -4));

         __ fstp_s(Address(rsp, 0));

         __ movflt(xmm0, Address(rsp, 0));

         __ lea(rsp, Address(rsp,  4));

       } else if (rt == T_DOUBLE) {

         __ lea(rsp, Address(rsp, -8));

         __ fstp_d(Address(rsp, 0));

         __ movdbl(xmm0, Address(rsp, 0));

         __ lea(rsp, Address(rsp,  8));

       }

     }

   %}

   enc_class pre_call_FPU %{

     // If method sets FPU control word restore it here

     debug_only(int off0 = cbuf.code_size());

     if( Compile::current()->in_24_bit_fp_mode() ) {

       MacroAssembler masm(&cbuf);

       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

     }

     debug_only(int off1 = cbuf.code_size());

     assert(off1 - off0 == pre_call_FPU_size(), "correct size prediction");

   %}

   enc_class post_call_FPU %{

     // If method sets FPU control word do it here also

     if( Compile::current()->in_24_bit_fp_mode() ) {

       MacroAssembler masm(&cbuf);

       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));

     }

   %}

   enc_class preserve_SP %{

     debug_only(int off0 = cbuf.code_size());

     MacroAssembler _masm(&cbuf);

     // RBP is preserved across all calls, even compiled calls.

     // Use it to preserve RSP in places where the callee might change the SP.

     __ movptr(rbp_mh_SP_save, rsp);

     debug_only(int off1 = cbuf.code_size());

     assert(off1 - off0 == preserve_SP_size(), "correct size prediction");

   %}

   enc_class restore_SP %{

     MacroAssembler _masm(&cbuf);

     __ movptr(rsp, rbp_mh_SP_save);

   %}

   enc_class Java_Static_Call (method meth) %{    // JAVA STATIC CALL

     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine

     // who we intended to call.

     cbuf.set_inst_mark();

     $$$emit8$primary;

     if ( !_method ) {

       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),

                      runtime_call_Relocation::spec(), RELOC_IMM32 );

     } else if(_optimized_virtual) {

       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),

                      opt_virtual_call_Relocation::spec(), RELOC_IMM32 );

     } else {

       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),

                      static_call_Relocation::spec(), RELOC_IMM32 );

     }

     if( _method ) {  // Emit stub for static call

       emit_java_to_interp(cbuf);

     }

   %}

   enc_class Java_Dynamic_Call (method meth) %{    // JAVA DYNAMIC CALL

     // !!!!!

     // Generate  "Mov EAX,0x00", placeholder instruction to load oop-info

     // emit_call_dynamic_prologue( cbuf );

     cbuf.set_inst_mark();

     emit_opcode(cbuf, 0xB8 + EAX_enc);        // mov    EAX,-1

     emit_d32_reloc(cbuf, (int)Universe::non_oop_word(), oop_Relocation::spec_for_immediate(), RELOC_IMM32);

     address  virtual_call_oop_addr = cbuf.inst_mark();

     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine

     // who we intended to call.

     cbuf.set_inst_mark();

     $$$emit8$primary;

     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),

                 virtual_call_Relocation::spec(virtual_call_oop_addr), RELOC_IMM32 );

   %}

   enc_class Java_Compiled_Call (method meth) %{    // JAVA COMPILED CALL

     int disp = in_bytes(methodOopDesc::from_compiled_offset());

     assert( -128 <= disp && disp <= 127, "compiled_code_offset isn't small");

     // CALL *[EAX+in_bytes(methodOopDesc::from_compiled_code_entry_point_offset())]

     cbuf.set_inst_mark();

     $$$emit8$primary;

     emit_rm(cbuf, 0x01, $secondary, EAX_enc );  // R/M byte

     emit_d8(cbuf, disp);             // Displacement

   %}

   enc_class Xor_Reg (eRegI dst) %{

     emit_opcode(cbuf, 0x33);

     emit_rm(cbuf, 0x3, $dst$$reg, $dst$$reg);

   %}

 //   Following encoding is no longer used, but may be restored if calling

 //   convention changes significantly.

 //   Became: Xor_Reg(EBP), Java_To_Runtime( labl )

 //

 //   enc_class Java_Interpreter_Call (label labl) %{    // JAVA INTERPRETER CALL

 //     // int ic_reg     = Matcher::inline_cache_reg();

 //     // int ic_encode  = Matcher::_regEncode[ic_reg];

 //     // int imo_reg    = Matcher::interpreter_method_oop_reg();

 //     // int imo_encode = Matcher::_regEncode[imo_reg];

 //

 //     // // Interpreter expects method_oop in EBX, currently a callee-saved register,

 //     // // so we load it immediately before the call

 //     // emit_opcode(cbuf, 0x8B);                     // MOV    imo_reg,ic_reg  # method_oop

 //     // emit_rm(cbuf, 0x03, imo_encode, ic_encode ); // R/M byte

 //

 //     // xor rbp,ebp

 //     emit_opcode(cbuf, 0x33);

 //     emit_rm(cbuf, 0x3, EBP_enc, EBP_enc);

 //

 //     // CALL to interpreter.

 //     cbuf.set_inst_mark();

 //     $$$emit8$primary;

 //     emit_d32_reloc(cbuf, ($labl$$label - (int)(cbuf.code_end()) - 4),

 //                 runtime_call_Relocation::spec(), RELOC_IMM32 );

 //   %}

   enc_class RegOpcImm (eRegI dst, immI8 shift) %{    // SHL, SAR, SHR

     $$$emit8$primary;

     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);

     $$$emit8$shift$$constant;

   %}

   enc_class LdImmI (eRegI dst, immI src) %{    // Load Immediate

     // Load immediate does not have a zero or sign extended version

     // for 8-bit immediates

     emit_opcode(cbuf, 0xB8 + $dst$$reg);

     $$$emit32$src$$constant;

   %}

   enc_class LdImmP (eRegI dst, immI src) %{    // Load Immediate

     // Load immediate does not have a zero or sign extended version

     // for 8-bit immediates

     emit_opcode(cbuf, $primary + $dst$$reg);

     $$$emit32$src$$constant;

   %}

   enc_class LdImmL_Lo( eRegL dst, immL src) %{    // Load Immediate

     // Load immediate does not have a zero or sign extended version

     // for 8-bit immediates

     int dst_enc = $dst$$reg;

     int src_con = $src$$constant & 0x0FFFFFFFFL;

     if (src_con == 0) {

       // xor dst, dst

       emit_opcode(cbuf, 0x33);

       emit_rm(cbuf, 0x3, dst_enc, dst_enc);

     } else {

       emit_opcode(cbuf, $primary + dst_enc);

       emit_d32(cbuf, src_con);

     }

   %}

   enc_class LdImmL_Hi( eRegL dst, immL src) %{    // Load Immediate

     // Load immediate does not have a zero or sign extended version

     // for 8-bit immediates

     int dst_enc = $dst$$reg + 2;

     int src_con = ((julong)($src$$constant)) >> 32;

     if (src_con == 0) {

       // xor dst, dst

       emit_opcode(cbuf, 0x33);

       emit_rm(cbuf, 0x3, dst_enc, dst_enc);

     } else {

       emit_opcode(cbuf, $primary + dst_enc);

       emit_d32(cbuf, src_con);

     }

   %}

   enc_class LdImmD (immD src) %{    // Load Immediate

     if( is_positive_zero_double($src$$constant)) {

       // FLDZ

       emit_opcode(cbuf,0xD9);

       emit_opcode(cbuf,0xEE);

     } else if( is_positive_one_double($src$$constant)) {

       // FLD1

       emit_opcode(cbuf,0xD9);

       emit_opcode(cbuf,0xE8);

     } else {

       emit_opcode(cbuf,0xDD);

       emit_rm(cbuf, 0x0, 0x0, 0x5);

       emit_double_constant(cbuf, $src$$constant);

     }

   %}

   enc_class LdImmF (immF src) %{    // Load Immediate

     if( is_positive_zero_float($src$$constant)) {

       emit_opcode(cbuf,0xD9);

       emit_opcode(cbuf,0xEE);

     } else if( is_positive_one_float($src$$constant)) {

       emit_opcode(cbuf,0xD9);

       emit_opcode(cbuf,0xE8);

     } else {

       $$$emit8$primary;

       // Load immediate does not have a zero or sign extended version

       // for 8-bit immediates

       // First load to TOS, then move to dst

       emit_rm(cbuf, 0x0, 0x0, 0x5);

       emit_float_constant(cbuf, $src$$constant);

     }

   %}

   enc_class LdImmX (regX dst, immXF con) %{    // Load Immediate

     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);

     emit_float_constant(cbuf, $con$$constant);

   %}

   enc_class LdImmXD (regXD dst, immXD con) %{    // Load Immediate

     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);

     emit_double_constant(cbuf, $con$$constant);

   %}

   enc_class load_conXD (regXD dst, immXD con) %{ // Load double constant

     // UseXmmLoadAndClearUpper ? movsd(dst, con) : movlpd(dst, con)

     emit_opcode(cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);

     emit_opcode(cbuf, 0x0F);

     emit_opcode(cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);

     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);

     emit_double_constant(cbuf, $con$$constant);

   %}

   enc_class Opc_MemImm_F(immF src) %{

     cbuf.set_inst_mark();

     $$$emit8$primary;

     emit_rm(cbuf, 0x0, $secondary, 0x5);

     emit_float_constant(cbuf, $src$$constant);

   %}

   enc_class MovI2X_reg(regX dst, eRegI src) %{

     emit_opcode(cbuf, 0x66 );     // MOVD dst,src

     emit_opcode(cbuf, 0x0F );

     emit_opcode(cbuf, 0x6E );

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class MovX2I_reg(eRegI dst, regX src) %{

     emit_opcode(cbuf, 0x66 );     // MOVD dst,src

     emit_opcode(cbuf, 0x0F );

     emit_opcode(cbuf, 0x7E );

     emit_rm(cbuf, 0x3, $src$$reg, $dst$$reg);

   %}

   enc_class MovL2XD_reg(regXD dst, eRegL src, regXD tmp) %{

     { // MOVD $dst,$src.lo

       emit_opcode(cbuf,0x66);

       emit_opcode(cbuf,0x0F);

       emit_opcode(cbuf,0x6E);

       emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

     }

     { // MOVD $tmp,$src.hi

       emit_opcode(cbuf,0x66);

       emit_opcode(cbuf,0x0F);

       emit_opcode(cbuf,0x6E);

       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg));

     }

     { // PUNPCKLDQ $dst,$tmp

       emit_opcode(cbuf,0x66);

       emit_opcode(cbuf,0x0F);

       emit_opcode(cbuf,0x62);

       emit_rm(cbuf, 0x3, $dst$$reg, $tmp$$reg);

      }

   %}

   enc_class MovXD2L_reg(eRegL dst, regXD src, regXD tmp) %{

     { // MOVD $dst.lo,$src

       emit_opcode(cbuf,0x66);

       emit_opcode(cbuf,0x0F);

       emit_opcode(cbuf,0x7E);

       emit_rm(cbuf, 0x3, $src$$reg, $dst$$reg);

     }

     { // PSHUFLW $tmp,$src,0x4E  (01001110b)

       emit_opcode(cbuf,0xF2);

       emit_opcode(cbuf,0x0F);

       emit_opcode(cbuf,0x70);

       emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);

       emit_d8(cbuf, 0x4E);

     }

     { // MOVD $dst.hi,$tmp

       emit_opcode(cbuf,0x66);

       emit_opcode(cbuf,0x0F);

       emit_opcode(cbuf,0x7E);

       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg));

     }

   %}

   // Encode a reg-reg copy.  If it is useless, then empty encoding.

   enc_class enc_Copy( eRegI dst, eRegI src ) %{

     encode_Copy( cbuf, $dst$$reg, $src$$reg );

   %}

   enc_class enc_CopyL_Lo( eRegI dst, eRegL src ) %{

     encode_Copy( cbuf, $dst$$reg, $src$$reg );

   %}

   // Encode xmm reg-reg copy.  If it is useless, then empty encoding.

   enc_class enc_CopyXD( RegXD dst, RegXD src ) %{

     encode_CopyXD( cbuf, $dst$$reg, $src$$reg );

   %}

   enc_class RegReg (eRegI dst, eRegI src) %{    // RegReg(Many)

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class RegReg_Lo(eRegL dst, eRegL src) %{    // RegReg(Many)

     $$$emit8$primary;

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class RegReg_Hi(eRegL dst, eRegL src) %{    // RegReg(Many)

     $$$emit8$secondary;

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));

   %}

   enc_class RegReg_Lo2(eRegL dst, eRegL src) %{    // RegReg(Many)

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class RegReg_Hi2(eRegL dst, eRegL src) %{    // RegReg(Many)

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));

   %}

   enc_class RegReg_HiLo( eRegL src, eRegI dst ) %{

     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($src$$reg));

   %}

   enc_class Con32 (immI src) %{    // Con32(storeImmI)

     // Output immediate

     $$$emit32$src$$constant;

   %}

   enc_class Con32F_as_bits(immF src) %{        // storeF_imm

     // Output Float immediate bits

     jfloat jf = $src$$constant;

     int    jf_as_bits = jint_cast( jf );

     emit_d32(cbuf, jf_as_bits);

   %}

   enc_class Con32XF_as_bits(immXF src) %{      // storeX_imm

     // Output Float immediate bits

     jfloat jf = $src$$constant;

     int    jf_as_bits = jint_cast( jf );

     emit_d32(cbuf, jf_as_bits);

   %}

   enc_class Con16 (immI src) %{    // Con16(storeImmI)

     // Output immediate

     $$$emit16$src$$constant;

   %}

   enc_class Con_d32(immI src) %{

     emit_d32(cbuf,$src$$constant);

   %}

   enc_class conmemref (eRegP t1) %{    // Con32(storeImmI)

     // Output immediate memory reference

     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );

     emit_d32(cbuf, 0x00);

   %}

   enc_class lock_prefix( ) %{

     if( os::is_MP() )

       emit_opcode(cbuf,0xF0);         // [Lock]

   %}

   // Cmp-xchg long value.

   // Note: we need to swap rbx, and rcx before and after the

   //       cmpxchg8 instruction because the instruction uses

   //       rcx as the high order word of the new value to store but

   //       our register encoding uses rbx,.

   enc_class enc_cmpxchg8(eSIRegP mem_ptr) %{

     // XCHG  rbx,ecx

     emit_opcode(cbuf,0x87);

     emit_opcode(cbuf,0xD9);

     // [Lock]

     if( os::is_MP() )

       emit_opcode(cbuf,0xF0);

     // CMPXCHG8 [Eptr]

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0xC7);

     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );

     // XCHG  rbx,ecx

     emit_opcode(cbuf,0x87);

     emit_opcode(cbuf,0xD9);

   %}

   enc_class enc_cmpxchg(eSIRegP mem_ptr) %{

     // [Lock]

     if( os::is_MP() )

       emit_opcode(cbuf,0xF0);

     // CMPXCHG [Eptr]

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0xB1);

     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );

   %}

   enc_class enc_flags_ne_to_boolean( iRegI res ) %{

     int res_encoding = $res$$reg;

     // MOV  res,0

     emit_opcode( cbuf, 0xB8 + res_encoding);

     emit_d32( cbuf, 0 );

     // JNE,s  fail

     emit_opcode(cbuf,0x75);

     emit_d8(cbuf, 5 );

     // MOV  res,1

     emit_opcode( cbuf, 0xB8 + res_encoding);

     emit_d32( cbuf, 1 );

     // fail:

   %}

   enc_class set_instruction_start( ) %{

     cbuf.set_inst_mark();            // Mark start of opcode for reloc info in mem operand

   %}

   enc_class RegMem (eRegI ereg, memory mem) %{    // emit_reg_mem

     int reg_encoding = $ereg$$reg;

     int base  = $mem$$base;

     int index = $mem$$index;

     int scale = $mem$$scale;

     int displace = $mem$$disp;

     bool disp_is_oop = $mem->disp_is_oop();

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);

   %}

   enc_class RegMem_Hi(eRegL ereg, memory mem) %{    // emit_reg_mem

     int reg_encoding = HIGH_FROM_LOW($ereg$$reg);  // Hi register of pair, computed from lo

     int base  = $mem$$base;

     int index = $mem$$index;

     int scale = $mem$$scale;

     int displace = $mem$$disp + 4;      // Offset is 4 further in memory

     assert( !$mem->disp_is_oop(), "Cannot add 4 to oop" );

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, false/*disp_is_oop*/);

   %}

   enc_class move_long_small_shift( eRegL dst, immI_1_31 cnt ) %{

     int r1, r2;

     if( $tertiary == 0xA4 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }

     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,$tertiary);

     emit_rm(cbuf, 0x3, r1, r2);

     emit_d8(cbuf,$cnt$$constant);

     emit_d8(cbuf,$primary);

     emit_rm(cbuf, 0x3, $secondary, r1);

     emit_d8(cbuf,$cnt$$constant);

   %}

   enc_class move_long_big_shift_sign( eRegL dst, immI_32_63 cnt ) %{

     emit_opcode( cbuf, 0x8B ); // Move

     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));

     emit_d8(cbuf,$primary);

     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);

     emit_d8(cbuf,$cnt$$constant-32);

     emit_d8(cbuf,$primary);

     emit_rm(cbuf, 0x3, $secondary, HIGH_FROM_LOW($dst$$reg));

     emit_d8(cbuf,31);

   %}

   enc_class move_long_big_shift_clr( eRegL dst, immI_32_63 cnt ) %{

     int r1, r2;

     if( $secondary == 0x5 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }

     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }

     emit_opcode( cbuf, 0x8B ); // Move r1,r2

     emit_rm(cbuf, 0x3, r1, r2);

     if( $cnt$$constant > 32 ) { // Shift, if not by zero

       emit_opcode(cbuf,$primary);

       emit_rm(cbuf, 0x3, $secondary, r1);

       emit_d8(cbuf,$cnt$$constant-32);

     }

     emit_opcode(cbuf,0x33);  // XOR r2,r2

     emit_rm(cbuf, 0x3, r2, r2);

   %}

   // Clone of RegMem but accepts an extra parameter to access each

   // half of a double in memory; it never needs relocation info.

   enc_class Mov_MemD_half_to_Reg (immI opcode, memory mem, immI disp_for_half, eRegI rm_reg) %{

     emit_opcode(cbuf,$opcode$$constant);

     int reg_encoding = $rm_reg$$reg;

     int base     = $mem$$base;

     int index    = $mem$$index;

     int scale    = $mem$$scale;

     int displace = $mem$$disp + $disp_for_half$$constant;

     bool disp_is_oop = false;

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);

   %}

   // !!!!! Special Custom Code used by MemMove, and stack access instructions !!!!!

   //

   // Clone of RegMem except the RM-byte's reg/opcode field is an ADLC-time constant

   // and it never needs relocation information.

   // Frequently used to move data between FPU's Stack Top and memory.

   enc_class RMopc_Mem_no_oop (immI rm_opcode, memory mem) %{

     int rm_byte_opcode = $rm_opcode$$constant;

     int base     = $mem$$base;

     int index    = $mem$$index;

     int scale    = $mem$$scale;

     int displace = $mem$$disp;

     assert( !$mem->disp_is_oop(), "No oops here because no relo info allowed" );

     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, false);

   %}

   enc_class RMopc_Mem (immI rm_opcode, memory mem) %{

     int rm_byte_opcode = $rm_opcode$$constant;

     int base     = $mem$$base;

     int index    = $mem$$index;

     int scale    = $mem$$scale;

     int displace = $mem$$disp;

     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals

     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_is_oop);

   %}

   enc_class RegLea (eRegI dst, eRegI src0, immI src1 ) %{    // emit_reg_lea

     int reg_encoding = $dst$$reg;

     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base

     int index        = 0x04;            // 0x04 indicates no index

     int scale        = 0x00;            // 0x00 indicates no scale

     int displace     = $src1$$constant; // 0x00 indicates no displacement

     bool disp_is_oop = false;

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);

   %}

   enc_class min_enc (eRegI dst, eRegI src) %{    // MIN

     // Compare dst,src

     emit_opcode(cbuf,0x3B);

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

     // jmp dst < src around move

     emit_opcode(cbuf,0x7C);

     emit_d8(cbuf,2);

     // move dst,src

     emit_opcode(cbuf,0x8B);

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class max_enc (eRegI dst, eRegI src) %{    // MAX

     // Compare dst,src

     emit_opcode(cbuf,0x3B);

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

     // jmp dst > src around move

     emit_opcode(cbuf,0x7F);

     emit_d8(cbuf,2);

     // move dst,src

     emit_opcode(cbuf,0x8B);

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class enc_FP_store(memory mem, regD src) %{

     // If src is FPR1, we can just FST to store it.

     // Else we need to FLD it to FPR1, then FSTP to store/pop it.

     int reg_encoding = 0x2; // Just store

     int base  = $mem$$base;

     int index = $mem$$index;

     int scale = $mem$$scale;

     int displace = $mem$$disp;

     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals

     if( $src$$reg != FPR1L_enc ) {

       reg_encoding = 0x3;  // Store & pop

       emit_opcode( cbuf, 0xD9 ); // FLD (i.e., push it)

       emit_d8( cbuf, 0xC0-1+$src$$reg );

     }

     cbuf.set_inst_mark();       // Mark start of opcode for reloc info in mem operand

     emit_opcode(cbuf,$primary);

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);

   %}

   enc_class neg_reg(eRegI dst) %{

     // NEG $dst

     emit_opcode(cbuf,0xF7);

     emit_rm(cbuf, 0x3, 0x03, $dst$$reg );

   %}

   enc_class setLT_reg(eCXRegI dst) %{

     // SETLT $dst

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0x9C);

     emit_rm( cbuf, 0x3, 0x4, $dst$$reg );

   %}

   enc_class enc_cmpLTP(ncxRegI p, ncxRegI q, ncxRegI y, eCXRegI tmp) %{    // cadd_cmpLT

     int tmpReg = $tmp$$reg;

     // SUB $p,$q

     emit_opcode(cbuf,0x2B);

     emit_rm(cbuf, 0x3, $p$$reg, $q$$reg);

     // SBB $tmp,$tmp

     emit_opcode(cbuf,0x1B);

     emit_rm(cbuf, 0x3, tmpReg, tmpReg);

     // AND $tmp,$y

     emit_opcode(cbuf,0x23);

     emit_rm(cbuf, 0x3, tmpReg, $y$$reg);

     // ADD $p,$tmp

     emit_opcode(cbuf,0x03);

     emit_rm(cbuf, 0x3, $p$$reg, tmpReg);

   %}

   enc_class enc_cmpLTP_mem(eRegI p, eRegI q, memory mem, eCXRegI tmp) %{    // cadd_cmpLT

     int tmpReg = $tmp$$reg;

     // SUB $p,$q

     emit_opcode(cbuf,0x2B);

     emit_rm(cbuf, 0x3, $p$$reg, $q$$reg);

     // SBB $tmp,$tmp

     emit_opcode(cbuf,0x1B);

     emit_rm(cbuf, 0x3, tmpReg, tmpReg);

     // AND $tmp,$y

     cbuf.set_inst_mark();       // Mark start of opcode for reloc info in mem operand

     emit_opcode(cbuf,0x23);

     int reg_encoding = tmpReg;

     int base  = $mem$$base;

     int index = $mem$$index;

     int scale = $mem$$scale;

     int displace = $mem$$disp;

     bool disp_is_oop = $mem->disp_is_oop();

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);

     // ADD $p,$tmp

     emit_opcode(cbuf,0x03);

     emit_rm(cbuf, 0x3, $p$$reg, tmpReg);

   %}

   enc_class shift_left_long( eRegL dst, eCXRegI shift ) %{

     // TEST shift,32

     emit_opcode(cbuf,0xF7);

     emit_rm(cbuf, 0x3, 0, ECX_enc);

     emit_d32(cbuf,0x20);

     // JEQ,s small

     emit_opcode(cbuf, 0x74);

     emit_d8(cbuf, 0x04);

     // MOV    $dst.hi,$dst.lo

     emit_opcode( cbuf, 0x8B );

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg );

     // CLR    $dst.lo

     emit_opcode(cbuf, 0x33);

     emit_rm(cbuf, 0x3, $dst$$reg, $dst$$reg);

 // small:

     // SHLD   $dst.hi,$dst.lo,$shift

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0xA5);

     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));

     // SHL    $dst.lo,$shift"

     emit_opcode(cbuf,0xD3);

     emit_rm(cbuf, 0x3, 0x4, $dst$$reg );

   %}

   enc_class shift_right_long( eRegL dst, eCXRegI shift ) %{

     // TEST shift,32

     emit_opcode(cbuf,0xF7);

     emit_rm(cbuf, 0x3, 0, ECX_enc);

     emit_d32(cbuf,0x20);

     // JEQ,s small

     emit_opcode(cbuf, 0x74);

     emit_d8(cbuf, 0x04);

     // MOV    $dst.lo,$dst.hi

     emit_opcode( cbuf, 0x8B );

     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );

     // CLR    $dst.hi

     emit_opcode(cbuf, 0x33);

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($dst$$reg));

 // small:

     // SHRD   $dst.lo,$dst.hi,$shift

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0xAD);

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);

     // SHR    $dst.hi,$shift"

     emit_opcode(cbuf,0xD3);

     emit_rm(cbuf, 0x3, 0x5, HIGH_FROM_LOW($dst$$reg) );

   %}

   enc_class shift_right_arith_long( eRegL dst, eCXRegI shift ) %{

     // TEST shift,32

     emit_opcode(cbuf,0xF7);

     emit_rm(cbuf, 0x3, 0, ECX_enc);

     emit_d32(cbuf,0x20);

     // JEQ,s small

     emit_opcode(cbuf, 0x74);

     emit_d8(cbuf, 0x05);

     // MOV    $dst.lo,$dst.hi

     emit_opcode( cbuf, 0x8B );

     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );

     // SAR    $dst.hi,31

     emit_opcode(cbuf, 0xC1);

     emit_rm(cbuf, 0x3, 7, HIGH_FROM_LOW($dst$$reg) );

     emit_d8(cbuf, 0x1F );

 // small:

     // SHRD   $dst.lo,$dst.hi,$shift

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0xAD);

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);

     // SAR    $dst.hi,$shift"

     emit_opcode(cbuf,0xD3);

     emit_rm(cbuf, 0x3, 0x7, HIGH_FROM_LOW($dst$$reg) );

   %}

   // ----------------- Encodings for floating point unit -----------------

   // May leave result in FPU-TOS or FPU reg depending on opcodes

   enc_class OpcReg_F (regF src) %{    // FMUL, FDIV

     $$$emit8$primary;

     emit_rm(cbuf, 0x3, $secondary, $src$$reg );

   %}

   // Pop argument in FPR0 with FSTP ST(0)

   enc_class PopFPU() %{

     emit_opcode( cbuf, 0xDD );

     emit_d8( cbuf, 0xD8 );

   %}

   // !!!!! equivalent to Pop_Reg_F

   enc_class Pop_Reg_D( regD dst ) %{

     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)

     emit_d8( cbuf, 0xD8+$dst$$reg );

   %}

   enc_class Push_Reg_D( regD dst ) %{

     emit_opcode( cbuf, 0xD9 );

     emit_d8( cbuf, 0xC0-1+$dst$$reg );   // FLD ST(i-1)

   %}

   enc_class strictfp_bias1( regD dst ) %{

     emit_opcode( cbuf, 0xDB );           // FLD m80real

     emit_opcode( cbuf, 0x2D );

     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias1() );

     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0

     emit_opcode( cbuf, 0xC8+$dst$$reg );

   %}

   enc_class strictfp_bias2( regD dst ) %{

     emit_opcode( cbuf, 0xDB );           // FLD m80real

     emit_opcode( cbuf, 0x2D );

     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias2() );

     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0

     emit_opcode( cbuf, 0xC8+$dst$$reg );

   %}

   // Special case for moving an integer register to a stack slot.

   enc_class OpcPRegSS( stackSlotI dst, eRegI src ) %{ // RegSS

     store_to_stackslot( cbuf, $primary, $src$$reg, $dst$$disp );

   %}

   // Special case for moving a register to a stack slot.

   enc_class RegSS( stackSlotI dst, eRegI src ) %{ // RegSS

     // Opcode already emitted

     emit_rm( cbuf, 0x02, $src$$reg, ESP_enc );   // R/M byte

     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);          // SIB byte

     emit_d32(cbuf, $dst$$disp);   // Displacement

   %}

   // Push the integer in stackSlot 'src' onto FP-stack

   enc_class Push_Mem_I( memory src ) %{    // FILD   [ESP+src]

     store_to_stackslot( cbuf, $primary, $secondary, $src$$disp );

   %}

   // Push the float in stackSlot 'src' onto FP-stack

   enc_class Push_Mem_F( memory src ) %{    // FLD_S   [ESP+src]

     store_to_stackslot( cbuf, 0xD9, 0x00, $src$$disp );

   %}

   // Push the double in stackSlot 'src' onto FP-stack

   enc_class Push_Mem_D( memory src ) %{    // FLD_D   [ESP+src]

     store_to_stackslot( cbuf, 0xDD, 0x00, $src$$disp );

   %}

   // Push FPU's TOS float to a stack-slot, and pop FPU-stack

   enc_class Pop_Mem_F( stackSlotF dst ) %{ // FSTP_S [ESP+dst]

     store_to_stackslot( cbuf, 0xD9, 0x03, $dst$$disp );

   %}

   // Same as Pop_Mem_F except for opcode

   // Push FPU's TOS double to a stack-slot, and pop FPU-stack

   enc_class Pop_Mem_D( stackSlotD dst ) %{ // FSTP_D [ESP+dst]

     store_to_stackslot( cbuf, 0xDD, 0x03, $dst$$disp );

   %}

   enc_class Pop_Reg_F( regF dst ) %{

     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)