Mercurial > jdk8-mips64-public > hotspot / file revision

 //

 // Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.

 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 //

 // This code is free software; you can redistribute it and/or modify it

 // under the terms of the GNU General Public License version 2 only, as

 // published by the Free Software Foundation.

 //

 // This code is distributed in the hope that it will be useful, but WITHOUT

 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 // version 2 for more details (a copy is included in the LICENSE file that

 // accompanied this code).

 //

 // You should have received a copy of the GNU General Public License version

 // 2 along with this work; if not, write to the Free Software Foundation,

 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 //

 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 // or visit www.oracle.com if you need additional information or have any

 // questions.

 //

 //

 // X86 Architecture Description File

 //----------REGISTER DEFINITION BLOCK------------------------------------------

 // This information is used by the matcher and the register allocator to

 // describe individual registers and classes of registers within the target

 // archtecture.

 register %{

 //----------Architecture Description Register Definitions----------------------

 // General Registers

 // "reg_def"  name ( register save type, C convention save type,

 //                   ideal register type, encoding );

 // Register Save Types:

 //

 // NS  = No-Save:       The register allocator assumes that these registers

 //                      can be used without saving upon entry to the method, &

 //                      that they do not need to be saved at call sites.

 //

 // SOC = Save-On-Call:  The register allocator assumes that these registers

 //                      can be used without saving upon entry to the method,

 //                      but that they must be saved at call sites.

 //

 // SOE = Save-On-Entry: The register allocator assumes that these registers

 //                      must be saved before using them upon entry to the

 //                      method, but they do not need to be saved at call

 //                      sites.

 //

 // AS  = Always-Save:   The register allocator assumes that these registers

 //                      must be saved before using them upon entry to the

 //                      method, & that they must be saved at call sites.

 //

 // Ideal Register Type is used to determine how to save & restore a

 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get

 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.

 //

 // The encoding number is the actual bit-pattern placed into the opcodes.

 // General Registers

 // Previously set EBX, ESI, and EDI as save-on-entry for java code

 // Turn off SOE in java-code due to frequent use of uncommon-traps.

 // Now that allocator is better, turn on ESI and EDI as SOE registers.

 reg_def EBX(SOC, SOE, Op_RegI, 3, rbx->as_VMReg());

 reg_def ECX(SOC, SOC, Op_RegI, 1, rcx->as_VMReg());

 reg_def ESI(SOC, SOE, Op_RegI, 6, rsi->as_VMReg());

 reg_def EDI(SOC, SOE, Op_RegI, 7, rdi->as_VMReg());

 // now that adapter frames are gone EBP is always saved and restored by the prolog/epilog code

 reg_def EBP(NS, SOE, Op_RegI, 5, rbp->as_VMReg());

 reg_def EDX(SOC, SOC, Op_RegI, 2, rdx->as_VMReg());

 reg_def EAX(SOC, SOC, Op_RegI, 0, rax->as_VMReg());

 reg_def ESP( NS,  NS, Op_RegI, 4, rsp->as_VMReg());

 // Float registers.  We treat TOS/FPR0 special.  It is invisible to the

 // allocator, and only shows up in the encodings.

 reg_def FPR0L( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());

 reg_def FPR0H( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());

 // Ok so here's the trick FPR1 is really st(0) except in the midst

 // of emission of assembly for a machnode. During the emission the fpu stack

 // is pushed making FPR1 == st(1) temporarily. However at any safepoint

 // the stack will not have this element so FPR1 == st(0) from the

 // oopMap viewpoint. This same weirdness with numbering causes

 // instruction encoding to have to play games with the register

 // encode to correct for this 0/1 issue. See MachSpillCopyNode::implementation

 // where it does flt->flt moves to see an example

 //

 reg_def FPR1L( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg());

 reg_def FPR1H( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg()->next());

 reg_def FPR2L( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg());

 reg_def FPR2H( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg()->next());

 reg_def FPR3L( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg());

 reg_def FPR3H( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg()->next());

 reg_def FPR4L( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg());

 reg_def FPR4H( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg()->next());

 reg_def FPR5L( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg());

 reg_def FPR5H( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg()->next());

 reg_def FPR6L( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg());

 reg_def FPR6H( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg()->next());

 reg_def FPR7L( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg());

 reg_def FPR7H( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg()->next());

 // Specify priority of register selection within phases of register

 // allocation.  Highest priority is first.  A useful heuristic is to

 // give registers a low priority when they are required by machine

 // instructions, like EAX and EDX.  Registers which are used as

 // pairs must fall on an even boundary (witness the FPR#L's in this list).

 // For the Intel integer registers, the equivalent Long pairs are

 // EDX:EAX, EBX:ECX, and EDI:EBP.

 alloc_class chunk0( ECX,   EBX,   EBP,   EDI,   EAX,   EDX,   ESI, ESP,

                     FPR0L, FPR0H, FPR1L, FPR1H, FPR2L, FPR2H,

                     FPR3L, FPR3H, FPR4L, FPR4H, FPR5L, FPR5H,

                     FPR6L, FPR6H, FPR7L, FPR7H );

 //----------Architecture Description Register Classes--------------------------

 // Several register classes are automatically defined based upon information in

 // this architecture description.

 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )

 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )

 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )

 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )

 //

 // Class for all registers

 reg_class any_reg(EAX, EDX, EBP, EDI, ESI, ECX, EBX, ESP);

 // Class for general registers

 reg_class int_reg(EAX, EDX, EBP, EDI, ESI, ECX, EBX);

 // Class for general registers which may be used for implicit null checks on win95

 // Also safe for use by tailjump. We don't want to allocate in rbp,

 reg_class int_reg_no_rbp(EAX, EDX, EDI, ESI, ECX, EBX);

 // Class of "X" registers

 reg_class int_x_reg(EBX, ECX, EDX, EAX);

 // Class of registers that can appear in an address with no offset.

 // EBP and ESP require an extra instruction byte for zero offset.

 // Used in fast-unlock

 reg_class p_reg(EDX, EDI, ESI, EBX);

 // Class for general registers not including ECX

 reg_class ncx_reg(EAX, EDX, EBP, EDI, ESI, EBX);

 // Class for general registers not including EAX

 reg_class nax_reg(EDX, EDI, ESI, ECX, EBX);

 // Class for general registers not including EAX or EBX.

 reg_class nabx_reg(EDX, EDI, ESI, ECX, EBP);

 // Class of EAX (for multiply and divide operations)

 reg_class eax_reg(EAX);

 // Class of EBX (for atomic add)

 reg_class ebx_reg(EBX);

 // Class of ECX (for shift and JCXZ operations and cmpLTMask)

 reg_class ecx_reg(ECX);

 // Class of EDX (for multiply and divide operations)

 reg_class edx_reg(EDX);

 // Class of EDI (for synchronization)

 reg_class edi_reg(EDI);

 // Class of ESI (for synchronization)

 reg_class esi_reg(ESI);

 // Singleton class for interpreter's stack pointer

 reg_class ebp_reg(EBP);

 // Singleton class for stack pointer

 reg_class sp_reg(ESP);

 // Singleton class for instruction pointer

 // reg_class ip_reg(EIP);

 // Class of integer register pairs

 reg_class long_reg( EAX,EDX, ECX,EBX, EBP,EDI );

 // Class of integer register pairs that aligns with calling convention

 reg_class eadx_reg( EAX,EDX );

 reg_class ebcx_reg( ECX,EBX );

 // Not AX or DX, used in divides

 reg_class nadx_reg( EBX,ECX,ESI,EDI,EBP );

 // Floating point registers.  Notice FPR0 is not a choice.

 // FPR0 is not ever allocated; we use clever encodings to fake

 // a 2-address instructions out of Intels FP stack.

 reg_class fp_flt_reg( FPR1L,FPR2L,FPR3L,FPR4L,FPR5L,FPR6L,FPR7L );

 reg_class fp_dbl_reg( FPR1L,FPR1H, FPR2L,FPR2H, FPR3L,FPR3H,

                       FPR4L,FPR4H, FPR5L,FPR5H, FPR6L,FPR6H,

                       FPR7L,FPR7H );

 reg_class fp_flt_reg0( FPR1L );

 reg_class fp_dbl_reg0( FPR1L,FPR1H );

 reg_class fp_dbl_reg1( FPR2L,FPR2H );

 reg_class fp_dbl_notreg0( FPR2L,FPR2H, FPR3L,FPR3H, FPR4L,FPR4H,

                           FPR5L,FPR5H, FPR6L,FPR6H, FPR7L,FPR7H );

 %}

 //----------SOURCE BLOCK-------------------------------------------------------

 // This is a block of C++ code which provides values, functions, and

 // definitions necessary in the rest of the architecture description

 source_hpp %{

 // Must be visible to the DFA in dfa_x86_32.cpp

 extern bool is_operand_hi32_zero(Node* n);

 %}

 source %{

 #define   RELOC_IMM32    Assembler::imm_operand

 #define   RELOC_DISP32   Assembler::disp32_operand

 #define __ _masm.

 // How to find the high register of a Long pair, given the low register

 #define   HIGH_FROM_LOW(x) ((x)+2)

 // These masks are used to provide 128-bit aligned bitmasks to the XMM

 // instructions, to allow sign-masking or sign-bit flipping.  They allow

 // fast versions of NegF/NegD and AbsF/AbsD.

 // Note: 'double' and 'long long' have 32-bits alignment on x86.

 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {

   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address

   // of 128-bits operands for SSE instructions.

   jlong *operand = (jlong*)(((uintptr_t)adr)&((uintptr_t)(~0xF)));

   // Store the value to a 128-bits operand.

   operand[0] = lo;

   operand[1] = hi;

   return operand;

 }

 // Buffer for 128-bits masks used by SSE instructions.

 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)

 // Static initialization during VM startup.

 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));

 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));

 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], CONST64(0x8000000080000000), CONST64(0x8000000080000000));

 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], CONST64(0x8000000000000000), CONST64(0x8000000000000000));

 // Offset hacking within calls.

 static int pre_call_resets_size() {

   int size = 0;

   Compile* C = Compile::current();

   if (C->in_24_bit_fp_mode()) {

     size += 6; // fldcw

   }

   if (C->max_vector_size() > 16) {

     size += 3; // vzeroupper

   }

   return size;

 }

 static int preserve_SP_size() {

   return 2;  // op, rm(reg/reg)

 }

 // !!!!! Special hack to get all type of calls to specify the byte offset

 //       from the start of the call to the point where the return address

 //       will point.

 int MachCallStaticJavaNode::ret_addr_offset() {

   int offset = 5 + pre_call_resets_size();  // 5 bytes from start of call to where return address points

   if (_method_handle_invoke)

     offset += preserve_SP_size();

   return offset;

 }

 int MachCallDynamicJavaNode::ret_addr_offset() {

   return 10 + pre_call_resets_size();  // 10 bytes from start of call to where return address points

 }

 static int sizeof_FFree_Float_Stack_All = -1;

 int MachCallRuntimeNode::ret_addr_offset() {

   assert(sizeof_FFree_Float_Stack_All != -1, "must have been emitted already");

   return sizeof_FFree_Float_Stack_All + 5 + pre_call_resets_size();

 }

 // Indicate if the safepoint node needs the polling page as an input.

 // Since x86 does have absolute addressing, it doesn't.

 bool SafePointNode::needs_polling_address_input() {

   return false;

 }

 //

 // Compute padding required for nodes which need alignment

 //

 // The address of the call instruction needs to be 4-byte aligned to

 // ensure that it does not span a cache line so that it can be patched.

 int CallStaticJavaDirectNode::compute_padding(int current_offset) const {

   current_offset += pre_call_resets_size();  // skip fldcw, if any

   current_offset += 1;      // skip call opcode byte

   return round_to(current_offset, alignment_required()) - current_offset;

 }

 // The address of the call instruction needs to be 4-byte aligned to

 // ensure that it does not span a cache line so that it can be patched.

 int CallStaticJavaHandleNode::compute_padding(int current_offset) const {

   current_offset += pre_call_resets_size();  // skip fldcw, if any

   current_offset += preserve_SP_size();   // skip mov rbp, rsp

   current_offset += 1;      // skip call opcode byte

   return round_to(current_offset, alignment_required()) - current_offset;

 }

 // The address of the call instruction needs to be 4-byte aligned to

 // ensure that it does not span a cache line so that it can be patched.

 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const {

   current_offset += pre_call_resets_size();  // skip fldcw, if any

   current_offset += 5;      // skip MOV instruction

   current_offset += 1;      // skip call opcode byte

   return round_to(current_offset, alignment_required()) - current_offset;

 }

 // EMIT_RM()

 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {

   unsigned char c = (unsigned char)((f1 << 6) | (f2 << 3) | f3);

   cbuf.insts()->emit_int8(c);

 }

 // EMIT_CC()

 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {

   unsigned char c = (unsigned char)( f1 | f2 );

   cbuf.insts()->emit_int8(c);

 }

 // EMIT_OPCODE()

 void emit_opcode(CodeBuffer &cbuf, int code) {

   cbuf.insts()->emit_int8((unsigned char) code);

 }

 // EMIT_OPCODE() w/ relocation information

 void emit_opcode(CodeBuffer &cbuf, int code, relocInfo::relocType reloc, int offset = 0) {

   cbuf.relocate(cbuf.insts_mark() + offset, reloc);

   emit_opcode(cbuf, code);

 }

 // EMIT_D8()

 void emit_d8(CodeBuffer &cbuf, int d8) {

   cbuf.insts()->emit_int8((unsigned char) d8);

 }

 // EMIT_D16()

 void emit_d16(CodeBuffer &cbuf, int d16) {

   cbuf.insts()->emit_int16(d16);

 }

 // EMIT_D32()

 void emit_d32(CodeBuffer &cbuf, int d32) {

   cbuf.insts()->emit_int32(d32);

 }

 // emit 32 bit value and construct relocation entry from relocInfo::relocType

 void emit_d32_reloc(CodeBuffer &cbuf, int d32, relocInfo::relocType reloc,

         int format) {

   cbuf.relocate(cbuf.insts_mark(), reloc, format);

   cbuf.insts()->emit_int32(d32);

 }

 // emit 32 bit value and construct relocation entry from RelocationHolder

 void emit_d32_reloc(CodeBuffer &cbuf, int d32, RelocationHolder const& rspec,

         int format) {

 #ifdef ASSERT

   if (rspec.reloc()->type() == relocInfo::oop_type && d32 != 0 && d32 != (int)Universe::non_oop_word()) {

     assert(cast_to_oop(d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d32)->is_scavengable()), "cannot embed scavengable oops in code");

   }

 #endif

   cbuf.relocate(cbuf.insts_mark(), rspec, format);

   cbuf.insts()->emit_int32(d32);

 }

 // Access stack slot for load or store

 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp) {

   emit_opcode( cbuf, opcode );               // (e.g., FILD   [ESP+src])

   if( -128 <= disp && disp <= 127 ) {

     emit_rm( cbuf, 0x01, rm_field, ESP_enc );  // R/M byte

     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte

     emit_d8 (cbuf, disp);     // Displacement  // R/M byte

   } else {

     emit_rm( cbuf, 0x02, rm_field, ESP_enc );  // R/M byte

     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte

     emit_d32(cbuf, disp);     // Displacement  // R/M byte

   }

 }

    // rRegI ereg, memory mem) %{    // emit_reg_mem

 void encode_RegMem( CodeBuffer &cbuf, int reg_encoding, int base, int index, int scale, int displace, relocInfo::relocType disp_reloc ) {

   // There is no index & no scale, use form without SIB byte

   if ((index == 0x4) &&

       (scale == 0) && (base != ESP_enc)) {

     // If no displacement, mode is 0x0; unless base is [EBP]

     if ( (displace == 0) && (base != EBP_enc) ) {

       emit_rm(cbuf, 0x0, reg_encoding, base);

     }

     else {                    // If 8-bit displacement, mode 0x1

       if ((displace >= -128) && (displace <= 127)

           && (disp_reloc == relocInfo::none) ) {

         emit_rm(cbuf, 0x1, reg_encoding, base);

         emit_d8(cbuf, displace);

       }

       else {                  // If 32-bit displacement

         if (base == -1) { // Special flag for absolute address

           emit_rm(cbuf, 0x0, reg_encoding, 0x5);

           // (manual lies; no SIB needed here)

           if ( disp_reloc != relocInfo::none ) {

             emit_d32_reloc(cbuf, displace, disp_reloc, 1);

           } else {

             emit_d32      (cbuf, displace);

           }

         }

         else {                // Normal base + offset

           emit_rm(cbuf, 0x2, reg_encoding, base);

           if ( disp_reloc != relocInfo::none ) {

             emit_d32_reloc(cbuf, displace, disp_reloc, 1);

           } else {

             emit_d32      (cbuf, displace);

           }

         }

       }

     }

   }

   else {                      // Else, encode with the SIB byte

     // If no displacement, mode is 0x0; unless base is [EBP]

     if (displace == 0 && (base != EBP_enc)) {  // If no displacement

       emit_rm(cbuf, 0x0, reg_encoding, 0x4);

       emit_rm(cbuf, scale, index, base);

     }

     else {                    // If 8-bit displacement, mode 0x1

       if ((displace >= -128) && (displace <= 127)

           && (disp_reloc == relocInfo::none) ) {

         emit_rm(cbuf, 0x1, reg_encoding, 0x4);

         emit_rm(cbuf, scale, index, base);

         emit_d8(cbuf, displace);

       }

       else {                  // If 32-bit displacement

         if (base == 0x04 ) {

           emit_rm(cbuf, 0x2, reg_encoding, 0x4);

           emit_rm(cbuf, scale, index, 0x04);

         } else {

           emit_rm(cbuf, 0x2, reg_encoding, 0x4);

           emit_rm(cbuf, scale, index, base);

         }

         if ( disp_reloc != relocInfo::none ) {

           emit_d32_reloc(cbuf, displace, disp_reloc, 1);

         } else {

           emit_d32      (cbuf, displace);

         }

       }

     }

   }

 }

 void encode_Copy( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {

   if( dst_encoding == src_encoding ) {

     // reg-reg copy, use an empty encoding

   } else {

     emit_opcode( cbuf, 0x8B );

     emit_rm(cbuf, 0x3, dst_encoding, src_encoding );

   }

 }

 void emit_cmpfp_fixup(MacroAssembler& _masm) {

   Label exit;

   __ jccb(Assembler::noParity, exit);

   __ pushf();

   //

   // comiss/ucomiss instructions set ZF,PF,CF flags and

   // zero OF,AF,SF for NaN values.

   // Fixup flags by zeroing ZF,PF so that compare of NaN

   // values returns 'less than' result (CF is set).

   // Leave the rest of flags unchanged.

   //

   //    7 6 5 4 3 2 1 0

   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)

   //    0 0 1 0 1 0 1 1   (0x2B)

   //

   __ andl(Address(rsp, 0), 0xffffff2b);

   __ popf();

   __ bind(exit);

 }

 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {

   Label done;

   __ movl(dst, -1);

   __ jcc(Assembler::parity, done);

   __ jcc(Assembler::below, done);

   __ setb(Assembler::notEqual, dst);

   __ movzbl(dst, dst);

   __ bind(done);

 }

 //=============================================================================

 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;

 int Compile::ConstantTable::calculate_table_base_offset() const {

   return 0;  // absolute addressing, no offset

 }

 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }

 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {

   ShouldNotReachHere();

 }

 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {

   // Empty encoding

 }

 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {

   return 0;

 }

 #ifndef PRODUCT

 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {

   st->print("# MachConstantBaseNode (empty encoding)");

 }

 #endif

 //=============================================================================

 #ifndef PRODUCT

 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {

   Compile* C = ra_->C;

   int framesize = C->frame_slots() << LogBytesPerInt;

   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");

   // Remove wordSize for return addr which is already pushed.

   framesize -= wordSize;

   if (C->need_stack_bang(framesize)) {

     framesize -= wordSize;

     st->print("# stack bang");

     st->print("\n\t");

     st->print("PUSH   EBP\t# Save EBP");

     if (framesize) {

       st->print("\n\t");

       st->print("SUB    ESP, #%d\t# Create frame",framesize);

     }

   } else {

     st->print("SUB    ESP, #%d\t# Create frame",framesize);

     st->print("\n\t");

     framesize -= wordSize;

     st->print("MOV    [ESP + #%d], EBP\t# Save EBP",framesize);

   }

   if (VerifyStackAtCalls) {

     st->print("\n\t");

     framesize -= wordSize;

     st->print("MOV    [ESP + #%d], 0xBADB100D\t# Majik cookie for stack depth check",framesize);

   }

   if( C->in_24_bit_fp_mode() ) {

     st->print("\n\t");

     st->print("FLDCW  \t# load 24 bit fpu control word");

   }

   if (UseSSE >= 2 && VerifyFPU) {

     st->print("\n\t");

     st->print("# verify FPU stack (must be clean on entry)");

   }

 #ifdef ASSERT

   if (VerifyStackAtCalls) {

     st->print("\n\t");

     st->print("# stack alignment check");

   }

 #endif

   st->cr();

 }

 #endif

 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

   Compile* C = ra_->C;

   MacroAssembler _masm(&cbuf);

   int framesize = C->frame_slots() << LogBytesPerInt;

   __ verified_entry(framesize, C->need_stack_bang(framesize), C->in_24_bit_fp_mode());

   C->set_frame_complete(cbuf.insts_size());

   if (C->has_mach_constant_base_node()) {

     // NOTE: We set the table base offset here because users might be

     // emitted before MachConstantBaseNode.

     Compile::ConstantTable& constant_table = C->constant_table();

     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());

   }

 }

 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {

   return MachNode::size(ra_); // too many variables; just compute it the hard way

 }

 int MachPrologNode::reloc() const {

   return 0; // a large enough number

 }

 //=============================================================================

 #ifndef PRODUCT

 void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {

   Compile *C = ra_->C;

   int framesize = C->frame_slots() << LogBytesPerInt;

   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");

   // Remove two words for return addr and rbp,

   framesize -= 2*wordSize;

   if (C->max_vector_size() > 16) {

     st->print("VZEROUPPER");

     st->cr(); st->print("\t");

   }

   if (C->in_24_bit_fp_mode()) {

     st->print("FLDCW  standard control word");

     st->cr(); st->print("\t");

   }

   if (framesize) {

     st->print("ADD    ESP,%d\t# Destroy frame",framesize);

     st->cr(); st->print("\t");

   }

   st->print_cr("POPL   EBP"); st->print("\t");

   if (do_polling() && C->is_method_compilation()) {

     st->print("TEST   PollPage,EAX\t! Poll Safepoint");

     st->cr(); st->print("\t");

   }

 }

 #endif

 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

   Compile *C = ra_->C;

   if (C->max_vector_size() > 16) {

     // Clear upper bits of YMM registers when current compiled code uses

     // wide vectors to avoid AVX <-> SSE transition penalty during call.

     MacroAssembler masm(&cbuf);

     masm.vzeroupper();

   }

   // If method set FPU control word, restore to standard control word

   if (C->in_24_bit_fp_mode()) {

     MacroAssembler masm(&cbuf);

     masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

   }

   int framesize = C->frame_slots() << LogBytesPerInt;

   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");

   // Remove two words for return addr and rbp,

   framesize -= 2*wordSize;

   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here

   if (framesize >= 128) {

     emit_opcode(cbuf, 0x81); // add  SP, #framesize

     emit_rm(cbuf, 0x3, 0x00, ESP_enc);

     emit_d32(cbuf, framesize);

   } else if (framesize) {

     emit_opcode(cbuf, 0x83); // add  SP, #framesize

     emit_rm(cbuf, 0x3, 0x00, ESP_enc);

     emit_d8(cbuf, framesize);

   }

   emit_opcode(cbuf, 0x58 | EBP_enc);

   if (do_polling() && C->is_method_compilation()) {

     cbuf.relocate(cbuf.insts_end(), relocInfo::poll_return_type, 0);

     emit_opcode(cbuf,0x85);

     emit_rm(cbuf, 0x0, EAX_enc, 0x5); // EAX

     emit_d32(cbuf, (intptr_t)os::get_polling_page());

   }

 }

 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {

   Compile *C = ra_->C;

   // If method set FPU control word, restore to standard control word

   int size = C->in_24_bit_fp_mode() ? 6 : 0;

   if (C->max_vector_size() > 16) size += 3; // vzeroupper

   if (do_polling() && C->is_method_compilation()) size += 6;

   int framesize = C->frame_slots() << LogBytesPerInt;

   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");

   // Remove two words for return addr and rbp,

   framesize -= 2*wordSize;

   size++; // popl rbp,

   if (framesize >= 128) {

     size += 6;

   } else {

     size += framesize ? 3 : 0;

   }

   return size;

 }

 int MachEpilogNode::reloc() const {

   return 0; // a large enough number

 }

 const Pipeline * MachEpilogNode::pipeline() const {

   return MachNode::pipeline_class();

 }

 int MachEpilogNode::safepoint_offset() const { return 0; }

 //=============================================================================

 enum RC { rc_bad, rc_int, rc_float, rc_xmm, rc_stack };

 static enum RC rc_class( OptoReg::Name reg ) {

   if( !OptoReg::is_valid(reg)  ) return rc_bad;

   if (OptoReg::is_stack(reg)) return rc_stack;

   VMReg r = OptoReg::as_VMReg(reg);

   if (r->is_Register()) return rc_int;

   if (r->is_FloatRegister()) {

     assert(UseSSE < 2, "shouldn't be used in SSE2+ mode");

     return rc_float;

   }

   assert(r->is_XMMRegister(), "must be");

   return rc_xmm;

 }

 static int impl_helper( CodeBuffer *cbuf, bool do_size, bool is_load, int offset, int reg,

                         int opcode, const char *op_str, int size, outputStream* st ) {

   if( cbuf ) {

     emit_opcode  (*cbuf, opcode );

     encode_RegMem(*cbuf, Matcher::_regEncode[reg], ESP_enc, 0x4, 0, offset, relocInfo::none);

 #ifndef PRODUCT

   } else if( !do_size ) {

     if( size != 0 ) st->print("\n\t");

     if( opcode == 0x8B || opcode == 0x89 ) { // MOV

       if( is_load ) st->print("%s   %s,[ESP + #%d]",op_str,Matcher::regName[reg],offset);

       else          st->print("%s   [ESP + #%d],%s",op_str,offset,Matcher::regName[reg]);

     } else { // FLD, FST, PUSH, POP

       st->print("%s [ESP + #%d]",op_str,offset);

     }

 #endif

   }

   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);

   return size+3+offset_size;

 }

 // Helper for XMM registers.  Extra opcode bits, limited syntax.

 static int impl_x_helper( CodeBuffer *cbuf, bool do_size, bool is_load,

                          int offset, int reg_lo, int reg_hi, int size, outputStream* st ) {

   if (cbuf) {

     MacroAssembler _masm(cbuf);

     if (reg_lo+1 == reg_hi) { // double move?

       if (is_load) {

         __ movdbl(as_XMMRegister(Matcher::_regEncode[reg_lo]), Address(rsp, offset));

       } else {

         __ movdbl(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[reg_lo]));

       }

     } else {

       if (is_load) {

         __ movflt(as_XMMRegister(Matcher::_regEncode[reg_lo]), Address(rsp, offset));

       } else {

         __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[reg_lo]));

       }

     }

 #ifndef PRODUCT

   } else if (!do_size) {

     if (size != 0) st->print("\n\t");

     if (reg_lo+1 == reg_hi) { // double move?

       if (is_load) st->print("%s %s,[ESP + #%d]",

                               UseXmmLoadAndClearUpper ? "MOVSD " : "MOVLPD",

                               Matcher::regName[reg_lo], offset);

       else         st->print("MOVSD  [ESP + #%d],%s",

                               offset, Matcher::regName[reg_lo]);

     } else {

       if (is_load) st->print("MOVSS  %s,[ESP + #%d]",

                               Matcher::regName[reg_lo], offset);

       else         st->print("MOVSS  [ESP + #%d],%s",

                               offset, Matcher::regName[reg_lo]);

     }

 #endif

   }

   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);

   // VEX_2bytes prefix is used if UseAVX > 0, so it takes the same 2 bytes as SIMD prefix.

   return size+5+offset_size;

 }

 static int impl_movx_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,

                             int src_hi, int dst_hi, int size, outputStream* st ) {

   if (cbuf) {

     MacroAssembler _masm(cbuf);

     if (src_lo+1 == src_hi && dst_lo+1 == dst_hi) { // double move?

       __ movdbl(as_XMMRegister(Matcher::_regEncode[dst_lo]),

                 as_XMMRegister(Matcher::_regEncode[src_lo]));

     } else {

       __ movflt(as_XMMRegister(Matcher::_regEncode[dst_lo]),

                 as_XMMRegister(Matcher::_regEncode[src_lo]));

     }

 #ifndef PRODUCT

   } else if (!do_size) {

     if (size != 0) st->print("\n\t");

     if (UseXmmRegToRegMoveAll) {//Use movaps,movapd to move between xmm registers

       if (src_lo+1 == src_hi && dst_lo+1 == dst_hi) { // double move?

         st->print("MOVAPD %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);

       } else {

         st->print("MOVAPS %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);

       }

     } else {

       if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?

         st->print("MOVSD  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);

       } else {

         st->print("MOVSS  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);

       }

     }

 #endif

   }

   // VEX_2bytes prefix is used if UseAVX > 0, and it takes the same 2 bytes as SIMD prefix.

   // Only MOVAPS SSE prefix uses 1 byte.

   int sz = 4;

   if (!(src_lo+1 == src_hi && dst_lo+1 == dst_hi) &&

       UseXmmRegToRegMoveAll && (UseAVX == 0)) sz = 3;

   return size + sz;

 }

 static int impl_movgpr2x_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,

                             int src_hi, int dst_hi, int size, outputStream* st ) {

   // 32-bit

   if (cbuf) {

     MacroAssembler _masm(cbuf);

     __ movdl(as_XMMRegister(Matcher::_regEncode[dst_lo]),

              as_Register(Matcher::_regEncode[src_lo]));

 #ifndef PRODUCT

   } else if (!do_size) {

     st->print("movdl   %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);

 #endif

   }

   return 4;

 }

 static int impl_movx2gpr_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,

                                  int src_hi, int dst_hi, int size, outputStream* st ) {

   // 32-bit

   if (cbuf) {

     MacroAssembler _masm(cbuf);

     __ movdl(as_Register(Matcher::_regEncode[dst_lo]),

              as_XMMRegister(Matcher::_regEncode[src_lo]));

 #ifndef PRODUCT

   } else if (!do_size) {

     st->print("movdl   %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);

 #endif

   }

   return 4;

 }

 static int impl_mov_helper( CodeBuffer *cbuf, bool do_size, int src, int dst, int size, outputStream* st ) {

   if( cbuf ) {

     emit_opcode(*cbuf, 0x8B );

     emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst], Matcher::_regEncode[src] );

 #ifndef PRODUCT

   } else if( !do_size ) {

     if( size != 0 ) st->print("\n\t");

     st->print("MOV    %s,%s",Matcher::regName[dst],Matcher::regName[src]);

 #endif

   }

   return size+2;

 }

 static int impl_fp_store_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int src_hi, int dst_lo, int dst_hi,

                                  int offset, int size, outputStream* st ) {

   if( src_lo != FPR1L_num ) {      // Move value to top of FP stack, if not already there

     if( cbuf ) {

       emit_opcode( *cbuf, 0xD9 );  // FLD (i.e., push it)

       emit_d8( *cbuf, 0xC0-1+Matcher::_regEncode[src_lo] );

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       st->print("FLD    %s",Matcher::regName[src_lo]);

 #endif

     }

     size += 2;

   }

   int st_op = (src_lo != FPR1L_num) ? EBX_num /*store & pop*/ : EDX_num /*store no pop*/;

   const char *op_str;

   int op;

   if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double store?

     op_str = (src_lo != FPR1L_num) ? "FSTP_D" : "FST_D ";

     op = 0xDD;

   } else {                   // 32-bit store

     op_str = (src_lo != FPR1L_num) ? "FSTP_S" : "FST_S ";

     op = 0xD9;

     assert( !OptoReg::is_valid(src_hi) && !OptoReg::is_valid(dst_hi), "no non-adjacent float-stores" );

   }

   return impl_helper(cbuf,do_size,false,offset,st_op,op,op_str,size, st);

 }

 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.

 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,

                           int src_hi, int dst_hi, uint ireg, outputStream* st);

 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,

                             int stack_offset, int reg, uint ireg, outputStream* st);

 static int vec_stack_to_stack_helper(CodeBuffer *cbuf, bool do_size, int src_offset,

                                      int dst_offset, uint ireg, outputStream* st) {

   int calc_size = 0;

   int src_offset_size = (src_offset == 0) ? 0 : ((src_offset < 0x80) ? 1 : 4);

   int dst_offset_size = (dst_offset == 0) ? 0 : ((dst_offset < 0x80) ? 1 : 4);

   switch (ireg) {

   case Op_VecS:

     calc_size = 3+src_offset_size + 3+dst_offset_size;

     break;

   case Op_VecD:

     calc_size = 3+src_offset_size + 3+dst_offset_size;

     src_offset += 4;

     dst_offset += 4;

     src_offset_size = (src_offset == 0) ? 0 : ((src_offset < 0x80) ? 1 : 4);

     dst_offset_size = (dst_offset == 0) ? 0 : ((dst_offset < 0x80) ? 1 : 4);

     calc_size += 3+src_offset_size + 3+dst_offset_size;

     break;

   case Op_VecX:

     calc_size = 6 + 6 + 5+src_offset_size + 5+dst_offset_size;

     break;

   case Op_VecY:

     calc_size = 6 + 6 + 5+src_offset_size + 5+dst_offset_size;

     break;

   default:

     ShouldNotReachHere();

   }

   if (cbuf) {

     MacroAssembler _masm(cbuf);

     int offset = __ offset();

     switch (ireg) {

     case Op_VecS:

       __ pushl(Address(rsp, src_offset));

       __ popl (Address(rsp, dst_offset));

       break;

     case Op_VecD:

       __ pushl(Address(rsp, src_offset));

       __ popl (Address(rsp, dst_offset));

       __ pushl(Address(rsp, src_offset+4));

       __ popl (Address(rsp, dst_offset+4));

       break;

     case Op_VecX:

       __ movdqu(Address(rsp, -16), xmm0);

       __ movdqu(xmm0, Address(rsp, src_offset));

       __ movdqu(Address(rsp, dst_offset), xmm0);

       __ movdqu(xmm0, Address(rsp, -16));

       break;

     case Op_VecY:

       __ vmovdqu(Address(rsp, -32), xmm0);

       __ vmovdqu(xmm0, Address(rsp, src_offset));

       __ vmovdqu(Address(rsp, dst_offset), xmm0);

       __ vmovdqu(xmm0, Address(rsp, -32));

       break;

     default:

       ShouldNotReachHere();

     }

     int size = __ offset() - offset;

     assert(size == calc_size, "incorrect size calculattion");

     return size;

 #ifndef PRODUCT

   } else if (!do_size) {

     switch (ireg) {

     case Op_VecS:

       st->print("pushl   [rsp + #%d]\t# 32-bit mem-mem spill\n\t"

                 "popl    [rsp + #%d]",

                 src_offset, dst_offset);

       break;

     case Op_VecD:

       st->print("pushl   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"

                 "popq    [rsp + #%d]\n\t"

                 "pushl   [rsp + #%d]\n\t"

                 "popq    [rsp + #%d]",

                 src_offset, dst_offset, src_offset+4, dst_offset+4);

       break;

      case Op_VecX:

       st->print("movdqu  [rsp - #16], xmm0\t# 128-bit mem-mem spill\n\t"

                 "movdqu  xmm0, [rsp + #%d]\n\t"

                 "movdqu  [rsp + #%d], xmm0\n\t"

                 "movdqu  xmm0, [rsp - #16]",

                 src_offset, dst_offset);

       break;

     case Op_VecY:

       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"

                 "vmovdqu xmm0, [rsp + #%d]\n\t"

                 "vmovdqu [rsp + #%d], xmm0\n\t"

                 "vmovdqu xmm0, [rsp - #32]",

                 src_offset, dst_offset);

       break;

     default:

       ShouldNotReachHere();

     }

 #endif

   }

   return calc_size;

 }

 uint MachSpillCopyNode::implementation( CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream* st ) const {

   // Get registers to move

   OptoReg::Name src_second = ra_->get_reg_second(in(1));

   OptoReg::Name src_first = ra_->get_reg_first(in(1));

   OptoReg::Name dst_second = ra_->get_reg_second(this );

   OptoReg::Name dst_first = ra_->get_reg_first(this );

   enum RC src_second_rc = rc_class(src_second);

   enum RC src_first_rc = rc_class(src_first);

   enum RC dst_second_rc = rc_class(dst_second);

   enum RC dst_first_rc = rc_class(dst_first);

   assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );

   // Generate spill code!

   int size = 0;

   if( src_first == dst_first && src_second == dst_second )

     return size;            // Self copy, no move

   if (bottom_type()->isa_vect() != NULL) {

     uint ireg = ideal_reg();

     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");

     assert((src_first_rc != rc_float && dst_first_rc != rc_float), "sanity");

     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY), "sanity");

     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {

       // mem -> mem

       int src_offset = ra_->reg2offset(src_first);

       int dst_offset = ra_->reg2offset(dst_first);

       return vec_stack_to_stack_helper(cbuf, do_size, src_offset, dst_offset, ireg, st);

     } else if (src_first_rc == rc_xmm && dst_first_rc == rc_xmm ) {

       return vec_mov_helper(cbuf, do_size, src_first, dst_first, src_second, dst_second, ireg, st);

     } else if (src_first_rc == rc_xmm && dst_first_rc == rc_stack ) {

       int stack_offset = ra_->reg2offset(dst_first);

       return vec_spill_helper(cbuf, do_size, false, stack_offset, src_first, ireg, st);

     } else if (src_first_rc == rc_stack && dst_first_rc == rc_xmm ) {

       int stack_offset = ra_->reg2offset(src_first);

       return vec_spill_helper(cbuf, do_size, true,  stack_offset, dst_first, ireg, st);

     } else {

       ShouldNotReachHere();

     }

   }

   // --------------------------------------

   // Check for mem-mem move.  push/pop to move.

   if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {

     if( src_second == dst_first ) { // overlapping stack copy ranges

       assert( src_second_rc == rc_stack && dst_second_rc == rc_stack, "we only expect a stk-stk copy here" );

       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);

       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);

       src_second_rc = dst_second_rc = rc_bad;  // flag as already moved the second bits

     }

     // move low bits

     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),ESI_num,0xFF,"PUSH  ",size, st);

     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),EAX_num,0x8F,"POP   ",size, st);

     if( src_second_rc == rc_stack && dst_second_rc == rc_stack ) { // mov second bits

       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);

       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);

     }

     return size;

   }

   // --------------------------------------

   // Check for integer reg-reg copy

   if( src_first_rc == rc_int && dst_first_rc == rc_int )

     size = impl_mov_helper(cbuf,do_size,src_first,dst_first,size, st);

   // Check for integer store

   if( src_first_rc == rc_int && dst_first_rc == rc_stack )

     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first,0x89,"MOV ",size, st);

   // Check for integer load

   if( dst_first_rc == rc_int && src_first_rc == rc_stack )

     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first,0x8B,"MOV ",size, st);

   // Check for integer reg-xmm reg copy

   if( src_first_rc == rc_int && dst_first_rc == rc_xmm ) {

     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),

             "no 64 bit integer-float reg moves" );

     return impl_movgpr2x_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);

   }

   // --------------------------------------

   // Check for float reg-reg copy

   if( src_first_rc == rc_float && dst_first_rc == rc_float ) {

     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||

             (src_first+1 == src_second && dst_first+1 == dst_second), "no non-adjacent float-moves" );

     if( cbuf ) {

       // Note the mucking with the register encode to compensate for the 0/1

       // indexing issue mentioned in a comment in the reg_def sections

       // for FPR registers many lines above here.

       if( src_first != FPR1L_num ) {

         emit_opcode  (*cbuf, 0xD9 );           // FLD    ST(i)

         emit_d8      (*cbuf, 0xC0+Matcher::_regEncode[src_first]-1 );

         emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)

         emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );

      } else {

         emit_opcode  (*cbuf, 0xDD );           // FST    ST(i)

         emit_d8      (*cbuf, 0xD0+Matcher::_regEncode[dst_first]-1 );

      }

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       if( src_first != FPR1L_num ) st->print("FLD    %s\n\tFSTP   %s",Matcher::regName[src_first],Matcher::regName[dst_first]);

       else                      st->print(             "FST    %s",                            Matcher::regName[dst_first]);

 #endif

     }

     return size + ((src_first != FPR1L_num) ? 2+2 : 2);

   }

   // Check for float store

   if( src_first_rc == rc_float && dst_first_rc == rc_stack ) {

     return impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,ra_->reg2offset(dst_first),size, st);

   }

   // Check for float load

   if( dst_first_rc == rc_float && src_first_rc == rc_stack ) {

     int offset = ra_->reg2offset(src_first);

     const char *op_str;

     int op;

     if( src_first+1 == src_second && dst_first+1 == dst_second ) { // double load?

       op_str = "FLD_D";

       op = 0xDD;

     } else {                   // 32-bit load

       op_str = "FLD_S";

       op = 0xD9;

       assert( src_second_rc == rc_bad && dst_second_rc == rc_bad, "no non-adjacent float-loads" );

     }

     if( cbuf ) {

       emit_opcode  (*cbuf, op );

       encode_RegMem(*cbuf, 0x0, ESP_enc, 0x4, 0, offset, relocInfo::none);

       emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)

       emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       st->print("%s  ST,[ESP + #%d]\n\tFSTP   %s",op_str, offset,Matcher::regName[dst_first]);

 #endif

     }

     int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);

     return size + 3+offset_size+2;

   }

   // Check for xmm reg-reg copy

   if( src_first_rc == rc_xmm && dst_first_rc == rc_xmm ) {

     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||

             (src_first+1 == src_second && dst_first+1 == dst_second),

             "no non-adjacent float-moves" );

     return impl_movx_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);

   }

   // Check for xmm reg-integer reg copy

   if( src_first_rc == rc_xmm && dst_first_rc == rc_int ) {

     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),

             "no 64 bit float-integer reg moves" );

     return impl_movx2gpr_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);

   }

   // Check for xmm store

   if( src_first_rc == rc_xmm && dst_first_rc == rc_stack ) {

     return impl_x_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first, src_second, size, st);

   }

   // Check for float xmm load

   if( dst_first_rc == rc_xmm && src_first_rc == rc_stack ) {

     return impl_x_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first, dst_second, size, st);

   }

   // Copy from float reg to xmm reg

   if( dst_first_rc == rc_xmm && src_first_rc == rc_float ) {

     // copy to the top of stack from floating point reg

     // and use LEA to preserve flags

     if( cbuf ) {

       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP-8]

       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);

       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);

       emit_d8(*cbuf,0xF8);

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       st->print("LEA    ESP,[ESP-8]");

 #endif

     }

     size += 4;

     size = impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,0,size, st);

     // Copy from the temp memory to the xmm reg.

     size = impl_x_helper(cbuf,do_size,true ,0,dst_first, dst_second, size, st);

     if( cbuf ) {

       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP+8]

       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);

       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);

       emit_d8(*cbuf,0x08);

 #ifndef PRODUCT

     } else if( !do_size ) {

       if( size != 0 ) st->print("\n\t");

       st->print("LEA    ESP,[ESP+8]");

 #endif

     }

     size += 4;

     return size;

   }

   assert( size > 0, "missed a case" );

   // --------------------------------------------------------------------

   // Check for second bits still needing moving.

   if( src_second == dst_second )

     return size;               // Self copy; no move

   assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );

   // Check for second word int-int move

   if( src_second_rc == rc_int && dst_second_rc == rc_int )

     return impl_mov_helper(cbuf,do_size,src_second,dst_second,size, st);

   // Check for second word integer store

   if( src_second_rc == rc_int && dst_second_rc == rc_stack )

     return impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),src_second,0x89,"MOV ",size, st);

   // Check for second word integer load

   if( dst_second_rc == rc_int && src_second_rc == rc_stack )

     return impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),dst_second,0x8B,"MOV ",size, st);

   Unimplemented();

 }

 #ifndef PRODUCT

 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {

   implementation( NULL, ra_, false, st );

 }

 #endif

 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

   implementation( &cbuf, ra_, false, NULL );

 }

 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {

   return implementation( NULL, ra_, true, NULL );

 }

 //=============================================================================

 #ifndef PRODUCT

 void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {

   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());

   int reg = ra_->get_reg_first(this);

   st->print("LEA    %s,[ESP + #%d]",Matcher::regName[reg],offset);

 }

 #endif

 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());

   int reg = ra_->get_encode(this);

   if( offset >= 128 ) {

     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]

     emit_rm(cbuf, 0x2, reg, 0x04);

     emit_rm(cbuf, 0x0, 0x04, ESP_enc);

     emit_d32(cbuf, offset);

   }

   else {

     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]

     emit_rm(cbuf, 0x1, reg, 0x04);

     emit_rm(cbuf, 0x0, 0x04, ESP_enc);

     emit_d8(cbuf, offset);

   }

 }

 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {

   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());

   if( offset >= 128 ) {

     return 7;

   }

   else {

     return 4;

   }

 }

 //=============================================================================

 #ifndef PRODUCT

 void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {

   st->print_cr(  "CMP    EAX,[ECX+4]\t# Inline cache check");

   st->print_cr("\tJNE    SharedRuntime::handle_ic_miss_stub");

   st->print_cr("\tNOP");

   st->print_cr("\tNOP");

   if( !OptoBreakpoint )

     st->print_cr("\tNOP");

 }

 #endif

 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {

   MacroAssembler masm(&cbuf);

 #ifdef ASSERT

   uint insts_size = cbuf.insts_size();

 #endif

   masm.cmpptr(rax, Address(rcx, oopDesc::klass_offset_in_bytes()));

   masm.jump_cc(Assembler::notEqual,

                RuntimeAddress(SharedRuntime::get_ic_miss_stub()));

   /* WARNING these NOPs are critical so that verified entry point is properly

      aligned for patching by NativeJump::patch_verified_entry() */

   int nops_cnt = 2;

   if( !OptoBreakpoint ) // Leave space for int3

      nops_cnt += 1;

   masm.nop(nops_cnt);

   assert(cbuf.insts_size() - insts_size == size(ra_), "checking code size of inline cache node");

 }

 uint MachUEPNode::size(PhaseRegAlloc *ra_) const {

   return OptoBreakpoint ? 11 : 12;

 }

 //=============================================================================

 uint size_exception_handler() {

   // NativeCall instruction size is the same as NativeJump.

   // exception handler starts out as jump and can be patched to

   // a call be deoptimization.  (4932387)

   // Note that this value is also credited (in output.cpp) to

   // the size of the code section.

   return NativeJump::instruction_size;

 }

 // Emit exception handler code.  Stuff framesize into a register

 // and call a VM stub routine.

 int emit_exception_handler(CodeBuffer& cbuf) {

   // Note that the code buffer's insts_mark is always relative to insts.

   // That's why we must use the macroassembler to generate a handler.

   MacroAssembler _masm(&cbuf);

   address base =

   __ start_a_stub(size_exception_handler());

   if (base == NULL)  return 0;  // CodeBuffer::expand failed

   int offset = __ offset();

   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));

   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");

   __ end_a_stub();

   return offset;

 }

 uint size_deopt_handler() {

   // NativeCall instruction size is the same as NativeJump.

   // exception handler starts out as jump and can be patched to

   // a call be deoptimization.  (4932387)

   // Note that this value is also credited (in output.cpp) to

   // the size of the code section.

   return 5 + NativeJump::instruction_size; // pushl(); jmp;

 }

 // Emit deopt handler code.

 int emit_deopt_handler(CodeBuffer& cbuf) {

   // Note that the code buffer's insts_mark is always relative to insts.

   // That's why we must use the macroassembler to generate a handler.

   MacroAssembler _masm(&cbuf);

   address base =

   __ start_a_stub(size_exception_handler());

   if (base == NULL)  return 0;  // CodeBuffer::expand failed

   int offset = __ offset();

   InternalAddress here(__ pc());

   __ pushptr(here.addr());

   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));

   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");

   __ end_a_stub();

   return offset;

 }

 int Matcher::regnum_to_fpu_offset(int regnum) {

   return regnum - 32; // The FP registers are in the second chunk

 }

 // This is UltraSparc specific, true just means we have fast l2f conversion

 const bool Matcher::convL2FSupported(void) {

   return true;

 }

 // Is this branch offset short enough that a short branch can be used?

 //

 // NOTE: If the platform does not provide any short branch variants, then

 //       this method should return false for offset 0.

 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {

   // The passed offset is relative to address of the branch.

   // On 86 a branch displacement is calculated relative to address

   // of a next instruction.

   offset -= br_size;

   // the short version of jmpConUCF2 contains multiple branches,

   // making the reach slightly less

   if (rule == jmpConUCF2_rule)

     return (-126 <= offset && offset <= 125);

   return (-128 <= offset && offset <= 127);

 }

 const bool Matcher::isSimpleConstant64(jlong value) {

   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.

   return false;

 }

 // The ecx parameter to rep stos for the ClearArray node is in dwords.

 const bool Matcher::init_array_count_is_in_bytes = false;

 // Threshold size for cleararray.

 const int Matcher::init_array_short_size = 8 * BytesPerLong;

 // Needs 2 CMOV's for longs.

 const int Matcher::long_cmove_cost() { return 1; }

 // No CMOVF/CMOVD with SSE/SSE2

 const int Matcher::float_cmove_cost() { return (UseSSE>=1) ? ConditionalMoveLimit : 0; }

 // Does the CPU require late expand (see block.cpp for description of late expand)?

 const bool Matcher::require_postalloc_expand = false;

 // Should the Matcher clone shifts on addressing modes, expecting them to

 // be subsumed into complex addressing expressions or compute them into

 // registers?  True for Intel but false for most RISCs

 const bool Matcher::clone_shift_expressions = true;

 // Do we need to mask the count passed to shift instructions or does

 // the cpu only look at the lower 5/6 bits anyway?

 const bool Matcher::need_masked_shift_count = false;

 bool Matcher::narrow_oop_use_complex_address() {

   ShouldNotCallThis();

   return true;

 }

 bool Matcher::narrow_klass_use_complex_address() {

   ShouldNotCallThis();

   return true;

 }

 // Is it better to copy float constants, or load them directly from memory?

 // Intel can load a float constant from a direct address, requiring no

 // extra registers.  Most RISCs will have to materialize an address into a

 // register first, so they would do better to copy the constant from stack.

 const bool Matcher::rematerialize_float_constants = true;

 // If CPU can load and store mis-aligned doubles directly then no fixup is

 // needed.  Else we split the double into 2 integer pieces and move it

 // piece-by-piece.  Only happens when passing doubles into C code as the

 // Java calling convention forces doubles to be aligned.

 const bool Matcher::misaligned_doubles_ok = true;

 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {

   // Get the memory operand from the node

   uint numopnds = node->num_opnds();        // Virtual call for number of operands

   uint skipped  = node->oper_input_base();  // Sum of leaves skipped so far

   assert( idx >= skipped, "idx too low in pd_implicit_null_fixup" );

   uint opcnt     = 1;                 // First operand

   uint num_edges = node->_opnds[1]->num_edges(); // leaves for first operand

   while( idx >= skipped+num_edges ) {

     skipped += num_edges;

     opcnt++;                          // Bump operand count

     assert( opcnt < numopnds, "Accessing non-existent operand" );

     num_edges = node->_opnds[opcnt]->num_edges(); // leaves for next operand

   }

   MachOper *memory = node->_opnds[opcnt];

   MachOper *new_memory = NULL;

   switch (memory->opcode()) {

   case DIRECT:

   case INDOFFSET32X:

     // No transformation necessary.

     return;

   case INDIRECT:

     new_memory = new (C) indirect_win95_safeOper( );

     break;

   case INDOFFSET8:

     new_memory = new (C) indOffset8_win95_safeOper(memory->disp(NULL, NULL, 0));

     break;

   case INDOFFSET32:

     new_memory = new (C) indOffset32_win95_safeOper(memory->disp(NULL, NULL, 0));

     break;

   case INDINDEXOFFSET:

     new_memory = new (C) indIndexOffset_win95_safeOper(memory->disp(NULL, NULL, 0));

     break;

   case INDINDEXSCALE:

     new_memory = new (C) indIndexScale_win95_safeOper(memory->scale());

     break;

   case INDINDEXSCALEOFFSET:

     new_memory = new (C) indIndexScaleOffset_win95_safeOper(memory->scale(), memory->disp(NULL, NULL, 0));

     break;

   case LOAD_LONG_INDIRECT:

   case LOAD_LONG_INDOFFSET32:

     // Does not use EBP as address register, use { EDX, EBX, EDI, ESI}

     return;

   default:

     assert(false, "unexpected memory operand in pd_implicit_null_fixup()");

     return;

   }

   node->_opnds[opcnt] = new_memory;

 }

 // Advertise here if the CPU requires explicit rounding operations

 // to implement the UseStrictFP mode.

 const bool Matcher::strict_fp_requires_explicit_rounding = true;

 // Are floats conerted to double when stored to stack during deoptimization?

 // On x32 it is stored with convertion only when FPU is used for floats.

 bool Matcher::float_in_double() { return (UseSSE == 0); }

 // Do ints take an entire long register or just half?

 const bool Matcher::int_in_long = false;

 // Return whether or not this register is ever used as an argument.  This

 // function is used on startup to build the trampoline stubs in generateOptoStub.

 // Registers not mentioned will be killed by the VM call in the trampoline, and

 // arguments in those registers not be available to the callee.

 bool Matcher::can_be_java_arg( int reg ) {

   if(  reg == ECX_num   || reg == EDX_num   ) return true;

   if( (reg == XMM0_num  || reg == XMM1_num ) && UseSSE>=1 ) return true;

   if( (reg == XMM0b_num || reg == XMM1b_num) && UseSSE>=2 ) return true;

   return false;

 }

 bool Matcher::is_spillable_arg( int reg ) {

   return can_be_java_arg(reg);

 }

 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {

   // Use hardware integer DIV instruction when

   // it is faster than a code which use multiply.

   // Only when constant divisor fits into 32 bit

   // (min_jint is excluded to get only correct

   // positive 32 bit values from negative).

   return VM_Version::has_fast_idiv() &&

          (divisor == (int)divisor && divisor != min_jint);

 }

 // Register for DIVI projection of divmodI

 RegMask Matcher::divI_proj_mask() {

   return EAX_REG_mask();

 }

 // Register for MODI projection of divmodI

 RegMask Matcher::modI_proj_mask() {

   return EDX_REG_mask();

 }

 // Register for DIVL projection of divmodL

 RegMask Matcher::divL_proj_mask() {

   ShouldNotReachHere();

   return RegMask();

 }

 // Register for MODL projection of divmodL

 RegMask Matcher::modL_proj_mask() {

   ShouldNotReachHere();

   return RegMask();

 }

 const RegMask Matcher::method_handle_invoke_SP_save_mask() {

   return EBP_REG_mask();

 }

 const RegMask Matcher::mathExactI_result_proj_mask() {

   return EAX_REG_mask();

 }

 const RegMask Matcher::mathExactL_result_proj_mask() {

   ShouldNotReachHere();

   return RegMask();

 }

 const RegMask Matcher::mathExactI_flags_proj_mask() {

   return INT_FLAGS_mask();

 }

 // Returns true if the high 32 bits of the value is known to be zero.

 bool is_operand_hi32_zero(Node* n) {

   int opc = n->Opcode();

   if (opc == Op_AndL) {

     Node* o2 = n->in(2);

     if (o2->is_Con() && (o2->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {

       return true;

     }

   }

   if (opc == Op_ConL && (n->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {

     return true;

   }

   return false;

 }

 %}

 //----------ENCODING BLOCK-----------------------------------------------------

 // This block specifies the encoding classes used by the compiler to output

 // byte streams.  Encoding classes generate functions which are called by

 // Machine Instruction Nodes in order to generate the bit encoding of the

 // instruction.  Operands specify their base encoding interface with the

 // interface keyword.  There are currently supported four interfaces,

 // REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER.  REG_INTER causes an

 // operand to generate a function which returns its register number when

 // queried.   CONST_INTER causes an operand to generate a function which

 // returns the value of the constant when queried.  MEMORY_INTER causes an

 // operand to generate four functions which return the Base Register, the

 // Index Register, the Scale Value, and the Offset Value of the operand when

 // queried.  COND_INTER causes an operand to generate six functions which

 // return the encoding code (ie - encoding bits for the instruction)

 // associated with each basic boolean condition for a conditional instruction.

 // Instructions specify two basic values for encoding.  They use the

 // ins_encode keyword to specify their encoding class (which must be one of

 // the class names specified in the encoding block), and they use the

 // opcode keyword to specify, in order, their primary, secondary, and

 // tertiary opcode.  Only the opcode sections which a particular instruction

 // needs for encoding need to be specified.

 encode %{

   // Build emit functions for each basic byte or larger field in the intel

   // encoding scheme (opcode, rm, sib, immediate), and call them from C++

   // code in the enc_class source block.  Emit functions will live in the

   // main source block for now.  In future, we can generalize this by

   // adding a syntax that specifies the sizes of fields in an order,

   // so that the adlc can build the emit functions automagically

   // Emit primary opcode

   enc_class OpcP %{

     emit_opcode(cbuf, $primary);

   %}

   // Emit secondary opcode

   enc_class OpcS %{

     emit_opcode(cbuf, $secondary);

   %}

   // Emit opcode directly

   enc_class Opcode(immI d8) %{

     emit_opcode(cbuf, $d8$$constant);

   %}

   enc_class SizePrefix %{

     emit_opcode(cbuf,0x66);

   %}

   enc_class RegReg (rRegI dst, rRegI src) %{    // RegReg(Many)

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class OpcRegReg (immI opcode, rRegI dst, rRegI src) %{    // OpcRegReg(Many)

     emit_opcode(cbuf,$opcode$$constant);

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class mov_r32_imm0( rRegI dst ) %{

     emit_opcode( cbuf, 0xB8 + $dst$$reg ); // 0xB8+ rd   -- MOV r32  ,imm32

     emit_d32   ( cbuf, 0x0  );             //                         imm32==0x0

   %}

   enc_class cdq_enc %{

     // Full implementation of Java idiv and irem; checks for

     // special case as described in JVM spec., p.243 & p.271.

     //

     //         normal case                           special case

     //

     // input : rax,: dividend                         min_int

     //         reg: divisor                          -1

     //

     // output: rax,: quotient  (= rax, idiv reg)       min_int

     //         rdx: remainder (= rax, irem reg)       0

     //

     //  Code sequnce:

     //

     //  81 F8 00 00 00 80    cmp         rax,80000000h

     //  0F 85 0B 00 00 00    jne         normal_case

     //  33 D2                xor         rdx,edx

     //  83 F9 FF             cmp         rcx,0FFh

     //  0F 84 03 00 00 00    je          done

     //                  normal_case:

     //  99                   cdq

     //  F7 F9                idiv        rax,ecx

     //                  done:

     //

     emit_opcode(cbuf,0x81); emit_d8(cbuf,0xF8);

     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);

     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x80);                     // cmp rax,80000000h

     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x85);

     emit_opcode(cbuf,0x0B); emit_d8(cbuf,0x00);

     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // jne normal_case

     emit_opcode(cbuf,0x33); emit_d8(cbuf,0xD2);                     // xor rdx,edx

     emit_opcode(cbuf,0x83); emit_d8(cbuf,0xF9); emit_d8(cbuf,0xFF); // cmp rcx,0FFh

     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x84);

     emit_opcode(cbuf,0x03); emit_d8(cbuf,0x00);

     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // je done

     // normal_case:

     emit_opcode(cbuf,0x99);                                         // cdq

     // idiv (note: must be emitted by the user of this rule)

     // normal:

   %}

   // Dense encoding for older common ops

   enc_class Opc_plus(immI opcode, rRegI reg) %{

     emit_opcode(cbuf, $opcode$$constant + $reg$$reg);

   %}

   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension

   enc_class OpcSE (immI imm) %{ // Emit primary opcode and set sign-extend bit

     // Check for 8-bit immediate, and set sign extend bit in opcode

     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {

       emit_opcode(cbuf, $primary | 0x02);

     }

     else {                          // If 32-bit immediate

       emit_opcode(cbuf, $primary);

     }

   %}

   enc_class OpcSErm (rRegI dst, immI imm) %{    // OpcSEr/m

     // Emit primary opcode and set sign-extend bit

     // Check for 8-bit immediate, and set sign extend bit in opcode

     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {

       emit_opcode(cbuf, $primary | 0x02);    }

     else {                          // If 32-bit immediate

       emit_opcode(cbuf, $primary);

     }

     // Emit r/m byte with secondary opcode, after primary opcode.

     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);

   %}

   enc_class Con8or32 (immI imm) %{    // Con8or32(storeImmI), 8 or 32 bits

     // Check for 8-bit immediate, and set sign extend bit in opcode

     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {

       $$$emit8$imm$$constant;

     }

     else {                          // If 32-bit immediate

       // Output immediate

       $$$emit32$imm$$constant;

     }

   %}

   enc_class Long_OpcSErm_Lo(eRegL dst, immL imm) %{

     // Emit primary opcode and set sign-extend bit

     // Check for 8-bit immediate, and set sign extend bit in opcode

     int con = (int)$imm$$constant; // Throw away top bits

     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);

     // Emit r/m byte with secondary opcode, after primary opcode.

     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);

     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);

     else                               emit_d32(cbuf,con);

   %}

   enc_class Long_OpcSErm_Hi(eRegL dst, immL imm) %{

     // Emit primary opcode and set sign-extend bit

     // Check for 8-bit immediate, and set sign extend bit in opcode

     int con = (int)($imm$$constant >> 32); // Throw away bottom bits

     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);

     // Emit r/m byte with tertiary opcode, after primary opcode.

     emit_rm(cbuf, 0x3, $tertiary, HIGH_FROM_LOW($dst$$reg));

     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);

     else                               emit_d32(cbuf,con);

   %}

   enc_class OpcSReg (rRegI dst) %{    // BSWAP

     emit_cc(cbuf, $secondary, $dst$$reg );

   %}

   enc_class bswap_long_bytes(eRegL dst) %{ // BSWAP

     int destlo = $dst$$reg;

     int desthi = HIGH_FROM_LOW(destlo);

     // bswap lo

     emit_opcode(cbuf, 0x0F);

     emit_cc(cbuf, 0xC8, destlo);

     // bswap hi

     emit_opcode(cbuf, 0x0F);

     emit_cc(cbuf, 0xC8, desthi);

     // xchg lo and hi

     emit_opcode(cbuf, 0x87);

     emit_rm(cbuf, 0x3, destlo, desthi);

   %}

   enc_class RegOpc (rRegI div) %{    // IDIV, IMOD, JMP indirect, ...

     emit_rm(cbuf, 0x3, $secondary, $div$$reg );

   %}

   enc_class enc_cmov(cmpOp cop ) %{ // CMOV

     $$$emit8$primary;

     emit_cc(cbuf, $secondary, $cop$$cmpcode);

   %}

   enc_class enc_cmov_dpr(cmpOp cop, regDPR src ) %{ // CMOV

     int op = 0xDA00 + $cop$$cmpcode + ($src$$reg-1);

     emit_d8(cbuf, op >> 8 );

     emit_d8(cbuf, op & 255);

   %}

   // emulate a CMOV with a conditional branch around a MOV

   enc_class enc_cmov_branch( cmpOp cop, immI brOffs ) %{ // CMOV

     // Invert sense of branch from sense of CMOV

     emit_cc( cbuf, 0x70, ($cop$$cmpcode^1) );

     emit_d8( cbuf, $brOffs$$constant );

   %}

   enc_class enc_PartialSubtypeCheck( ) %{

     Register Redi = as_Register(EDI_enc); // result register

     Register Reax = as_Register(EAX_enc); // super class

     Register Recx = as_Register(ECX_enc); // killed

     Register Resi = as_Register(ESI_enc); // sub class

     Label miss;

     MacroAssembler _masm(&cbuf);

     __ check_klass_subtype_slow_path(Resi, Reax, Recx, Redi,

                                      NULL, &miss,

                                      /*set_cond_codes:*/ true);

     if ($primary) {

       __ xorptr(Redi, Redi);

     }

     __ bind(miss);

   %}

   enc_class FFree_Float_Stack_All %{    // Free_Float_Stack_All

     MacroAssembler masm(&cbuf);

     int start = masm.offset();

     if (UseSSE >= 2) {

       if (VerifyFPU) {

         masm.verify_FPU(0, "must be empty in SSE2+ mode");

       }

     } else {

       // External c_calling_convention expects the FPU stack to be 'clean'.

       // Compiled code leaves it dirty.  Do cleanup now.

       masm.empty_FPU_stack();

     }

     if (sizeof_FFree_Float_Stack_All == -1) {

       sizeof_FFree_Float_Stack_All = masm.offset() - start;

     } else {

       assert(masm.offset() - start == sizeof_FFree_Float_Stack_All, "wrong size");

     }

   %}

   enc_class Verify_FPU_For_Leaf %{

     if( VerifyFPU ) {

       MacroAssembler masm(&cbuf);

       masm.verify_FPU( -3, "Returning from Runtime Leaf call");

     }

   %}

   enc_class Java_To_Runtime (method meth) %{    // CALL Java_To_Runtime, Java_To_Runtime_Leaf

     // This is the instruction starting address for relocation info.

     cbuf.set_insts_mark();

     $$$emit8$primary;

     // CALL directly to the runtime

     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),

                 runtime_call_Relocation::spec(), RELOC_IMM32 );

     if (UseSSE >= 2) {

       MacroAssembler _masm(&cbuf);

       BasicType rt = tf()->return_type();

       if ((rt == T_FLOAT || rt == T_DOUBLE) && !return_value_is_used()) {

         // A C runtime call where the return value is unused.  In SSE2+

         // mode the result needs to be removed from the FPU stack.  It's

         // likely that this function call could be removed by the

         // optimizer if the C function is a pure function.

         __ ffree(0);

       } else if (rt == T_FLOAT) {

         __ lea(rsp, Address(rsp, -4));

         __ fstp_s(Address(rsp, 0));

         __ movflt(xmm0, Address(rsp, 0));

         __ lea(rsp, Address(rsp,  4));

       } else if (rt == T_DOUBLE) {

         __ lea(rsp, Address(rsp, -8));

         __ fstp_d(Address(rsp, 0));

         __ movdbl(xmm0, Address(rsp, 0));

         __ lea(rsp, Address(rsp,  8));

       }

     }

   %}

   enc_class pre_call_resets %{

     // If method sets FPU control word restore it here

     debug_only(int off0 = cbuf.insts_size());

     if (ra_->C->in_24_bit_fp_mode()) {

       MacroAssembler _masm(&cbuf);

       __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

     }

     if (ra_->C->max_vector_size() > 16) {

       // Clear upper bits of YMM registers when current compiled code uses

       // wide vectors to avoid AVX <-> SSE transition penalty during call.

       MacroAssembler _masm(&cbuf);

       __ vzeroupper();

     }

     debug_only(int off1 = cbuf.insts_size());

     assert(off1 - off0 == pre_call_resets_size(), "correct size prediction");

   %}

   enc_class post_call_FPU %{

     // If method sets FPU control word do it here also

     if (Compile::current()->in_24_bit_fp_mode()) {

       MacroAssembler masm(&cbuf);

       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));

     }

   %}

   enc_class Java_Static_Call (method meth) %{    // JAVA STATIC CALL

     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine

     // who we intended to call.

     cbuf.set_insts_mark();

     $$$emit8$primary;

     if (!_method) {

       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),

                      runtime_call_Relocation::spec(), RELOC_IMM32 );

     } else if (_optimized_virtual) {

       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),

                      opt_virtual_call_Relocation::spec(), RELOC_IMM32 );

     } else {

       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),

                      static_call_Relocation::spec(), RELOC_IMM32 );

     }

     if (_method) {  // Emit stub for static call.

       CompiledStaticCall::emit_to_interp_stub(cbuf);

     }

   %}

   enc_class Java_Dynamic_Call (method meth) %{    // JAVA DYNAMIC CALL

     MacroAssembler _masm(&cbuf);

     __ ic_call((address)$meth$$method);

   %}

   enc_class Java_Compiled_Call (method meth) %{    // JAVA COMPILED CALL

     int disp = in_bytes(Method::from_compiled_offset());

     assert( -128 <= disp && disp <= 127, "compiled_code_offset isn't small");

     // CALL *[EAX+in_bytes(Method::from_compiled_code_entry_point_offset())]

     cbuf.set_insts_mark();

     $$$emit8$primary;

     emit_rm(cbuf, 0x01, $secondary, EAX_enc );  // R/M byte

     emit_d8(cbuf, disp);             // Displacement

   %}

 //   Following encoding is no longer used, but may be restored if calling

 //   convention changes significantly.

 //   Became: Xor_Reg(EBP), Java_To_Runtime( labl )

 //

 //   enc_class Java_Interpreter_Call (label labl) %{    // JAVA INTERPRETER CALL

 //     // int ic_reg     = Matcher::inline_cache_reg();

 //     // int ic_encode  = Matcher::_regEncode[ic_reg];

 //     // int imo_reg    = Matcher::interpreter_method_oop_reg();

 //     // int imo_encode = Matcher::_regEncode[imo_reg];

 //

 //     // // Interpreter expects method_oop in EBX, currently a callee-saved register,

 //     // // so we load it immediately before the call

 //     // emit_opcode(cbuf, 0x8B);                     // MOV    imo_reg,ic_reg  # method_oop

 //     // emit_rm(cbuf, 0x03, imo_encode, ic_encode ); // R/M byte

 //

 //     // xor rbp,ebp

 //     emit_opcode(cbuf, 0x33);

 //     emit_rm(cbuf, 0x3, EBP_enc, EBP_enc);

 //

 //     // CALL to interpreter.

 //     cbuf.set_insts_mark();

 //     $$$emit8$primary;

 //     emit_d32_reloc(cbuf, ($labl$$label - (int)(cbuf.insts_end()) - 4),

 //                 runtime_call_Relocation::spec(), RELOC_IMM32 );

 //   %}

   enc_class RegOpcImm (rRegI dst, immI8 shift) %{    // SHL, SAR, SHR

     $$$emit8$primary;

     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);

     $$$emit8$shift$$constant;

   %}

   enc_class LdImmI (rRegI dst, immI src) %{    // Load Immediate

     // Load immediate does not have a zero or sign extended version

     // for 8-bit immediates

     emit_opcode(cbuf, 0xB8 + $dst$$reg);

     $$$emit32$src$$constant;

   %}

   enc_class LdImmP (rRegI dst, immI src) %{    // Load Immediate

     // Load immediate does not have a zero or sign extended version

     // for 8-bit immediates

     emit_opcode(cbuf, $primary + $dst$$reg);

     $$$emit32$src$$constant;

   %}

   enc_class LdImmL_Lo( eRegL dst, immL src) %{    // Load Immediate

     // Load immediate does not have a zero or sign extended version

     // for 8-bit immediates

     int dst_enc = $dst$$reg;

     int src_con = $src$$constant & 0x0FFFFFFFFL;

     if (src_con == 0) {

       // xor dst, dst

       emit_opcode(cbuf, 0x33);

       emit_rm(cbuf, 0x3, dst_enc, dst_enc);

     } else {

       emit_opcode(cbuf, $primary + dst_enc);

       emit_d32(cbuf, src_con);

     }

   %}

   enc_class LdImmL_Hi( eRegL dst, immL src) %{    // Load Immediate

     // Load immediate does not have a zero or sign extended version

     // for 8-bit immediates

     int dst_enc = $dst$$reg + 2;

     int src_con = ((julong)($src$$constant)) >> 32;

     if (src_con == 0) {

       // xor dst, dst

       emit_opcode(cbuf, 0x33);

       emit_rm(cbuf, 0x3, dst_enc, dst_enc);

     } else {

       emit_opcode(cbuf, $primary + dst_enc);

       emit_d32(cbuf, src_con);

     }

   %}

   // Encode a reg-reg copy.  If it is useless, then empty encoding.

   enc_class enc_Copy( rRegI dst, rRegI src ) %{

     encode_Copy( cbuf, $dst$$reg, $src$$reg );

   %}

   enc_class enc_CopyL_Lo( rRegI dst, eRegL src ) %{

     encode_Copy( cbuf, $dst$$reg, $src$$reg );

   %}

   enc_class RegReg (rRegI dst, rRegI src) %{    // RegReg(Many)

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class RegReg_Lo(eRegL dst, eRegL src) %{    // RegReg(Many)

     $$$emit8$primary;

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class RegReg_Hi(eRegL dst, eRegL src) %{    // RegReg(Many)

     $$$emit8$secondary;

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));

   %}

   enc_class RegReg_Lo2(eRegL dst, eRegL src) %{    // RegReg(Many)

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class RegReg_Hi2(eRegL dst, eRegL src) %{    // RegReg(Many)

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));

   %}

   enc_class RegReg_HiLo( eRegL src, rRegI dst ) %{

     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($src$$reg));

   %}

   enc_class Con32 (immI src) %{    // Con32(storeImmI)

     // Output immediate

     $$$emit32$src$$constant;

   %}

   enc_class Con32FPR_as_bits(immFPR src) %{        // storeF_imm

     // Output Float immediate bits

     jfloat jf = $src$$constant;

     int    jf_as_bits = jint_cast( jf );

     emit_d32(cbuf, jf_as_bits);

   %}

   enc_class Con32F_as_bits(immF src) %{      // storeX_imm

     // Output Float immediate bits

     jfloat jf = $src$$constant;

     int    jf_as_bits = jint_cast( jf );

     emit_d32(cbuf, jf_as_bits);

   %}

   enc_class Con16 (immI src) %{    // Con16(storeImmI)

     // Output immediate

     $$$emit16$src$$constant;

   %}

   enc_class Con_d32(immI src) %{

     emit_d32(cbuf,$src$$constant);

   %}

   enc_class conmemref (eRegP t1) %{    // Con32(storeImmI)

     // Output immediate memory reference

     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );

     emit_d32(cbuf, 0x00);

   %}

   enc_class lock_prefix( ) %{

     if( os::is_MP() )

       emit_opcode(cbuf,0xF0);         // [Lock]

   %}

   // Cmp-xchg long value.

   // Note: we need to swap rbx, and rcx before and after the

   //       cmpxchg8 instruction because the instruction uses

   //       rcx as the high order word of the new value to store but

   //       our register encoding uses rbx,.

   enc_class enc_cmpxchg8(eSIRegP mem_ptr) %{

     // XCHG  rbx,ecx

     emit_opcode(cbuf,0x87);

     emit_opcode(cbuf,0xD9);

     // [Lock]

     if( os::is_MP() )

       emit_opcode(cbuf,0xF0);

     // CMPXCHG8 [Eptr]

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0xC7);

     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );

     // XCHG  rbx,ecx

     emit_opcode(cbuf,0x87);

     emit_opcode(cbuf,0xD9);

   %}

   enc_class enc_cmpxchg(eSIRegP mem_ptr) %{

     // [Lock]

     if( os::is_MP() )

       emit_opcode(cbuf,0xF0);

     // CMPXCHG [Eptr]

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0xB1);

     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );

   %}

   enc_class enc_flags_ne_to_boolean( iRegI res ) %{

     int res_encoding = $res$$reg;

     // MOV  res,0

     emit_opcode( cbuf, 0xB8 + res_encoding);

     emit_d32( cbuf, 0 );

     // JNE,s  fail

     emit_opcode(cbuf,0x75);

     emit_d8(cbuf, 5 );

     // MOV  res,1

     emit_opcode( cbuf, 0xB8 + res_encoding);

     emit_d32( cbuf, 1 );

     // fail:

   %}

   enc_class set_instruction_start( ) %{

     cbuf.set_insts_mark();            // Mark start of opcode for reloc info in mem operand

   %}

   enc_class RegMem (rRegI ereg, memory mem) %{    // emit_reg_mem

     int reg_encoding = $ereg$$reg;

     int base  = $mem$$base;

     int index = $mem$$index;

     int scale = $mem$$scale;

     int displace = $mem$$disp;

     relocInfo::relocType disp_reloc = $mem->disp_reloc();

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_reloc);

   %}

   enc_class RegMem_Hi(eRegL ereg, memory mem) %{    // emit_reg_mem

     int reg_encoding = HIGH_FROM_LOW($ereg$$reg);  // Hi register of pair, computed from lo

     int base  = $mem$$base;

     int index = $mem$$index;

     int scale = $mem$$scale;

     int displace = $mem$$disp + 4;      // Offset is 4 further in memory

     assert( $mem->disp_reloc() == relocInfo::none, "Cannot add 4 to oop" );

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, relocInfo::none);

   %}

   enc_class move_long_small_shift( eRegL dst, immI_1_31 cnt ) %{

     int r1, r2;

     if( $tertiary == 0xA4 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }

     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,$tertiary);

     emit_rm(cbuf, 0x3, r1, r2);

     emit_d8(cbuf,$cnt$$constant);

     emit_d8(cbuf,$primary);

     emit_rm(cbuf, 0x3, $secondary, r1);

     emit_d8(cbuf,$cnt$$constant);

   %}

   enc_class move_long_big_shift_sign( eRegL dst, immI_32_63 cnt ) %{

     emit_opcode( cbuf, 0x8B ); // Move

     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));

     if( $cnt$$constant > 32 ) { // Shift, if not by zero

       emit_d8(cbuf,$primary);

       emit_rm(cbuf, 0x3, $secondary, $dst$$reg);

       emit_d8(cbuf,$cnt$$constant-32);

     }

     emit_d8(cbuf,$primary);

     emit_rm(cbuf, 0x3, $secondary, HIGH_FROM_LOW($dst$$reg));

     emit_d8(cbuf,31);

   %}

   enc_class move_long_big_shift_clr( eRegL dst, immI_32_63 cnt ) %{

     int r1, r2;

     if( $secondary == 0x5 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }

     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }

     emit_opcode( cbuf, 0x8B ); // Move r1,r2

     emit_rm(cbuf, 0x3, r1, r2);

     if( $cnt$$constant > 32 ) { // Shift, if not by zero

       emit_opcode(cbuf,$primary);

       emit_rm(cbuf, 0x3, $secondary, r1);

       emit_d8(cbuf,$cnt$$constant-32);

     }

     emit_opcode(cbuf,0x33);  // XOR r2,r2

     emit_rm(cbuf, 0x3, r2, r2);

   %}

   // Clone of RegMem but accepts an extra parameter to access each

   // half of a double in memory; it never needs relocation info.

   enc_class Mov_MemD_half_to_Reg (immI opcode, memory mem, immI disp_for_half, rRegI rm_reg) %{

     emit_opcode(cbuf,$opcode$$constant);

     int reg_encoding = $rm_reg$$reg;

     int base     = $mem$$base;

     int index    = $mem$$index;

     int scale    = $mem$$scale;

     int displace = $mem$$disp + $disp_for_half$$constant;

     relocInfo::relocType disp_reloc = relocInfo::none;

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_reloc);

   %}

   // !!!!! Special Custom Code used by MemMove, and stack access instructions !!!!!

   //

   // Clone of RegMem except the RM-byte's reg/opcode field is an ADLC-time constant

   // and it never needs relocation information.

   // Frequently used to move data between FPU's Stack Top and memory.

   enc_class RMopc_Mem_no_oop (immI rm_opcode, memory mem) %{

     int rm_byte_opcode = $rm_opcode$$constant;

     int base     = $mem$$base;

     int index    = $mem$$index;

     int scale    = $mem$$scale;

     int displace = $mem$$disp;

     assert( $mem->disp_reloc() == relocInfo::none, "No oops here because no reloc info allowed" );

     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, relocInfo::none);

   %}

   enc_class RMopc_Mem (immI rm_opcode, memory mem) %{

     int rm_byte_opcode = $rm_opcode$$constant;

     int base     = $mem$$base;

     int index    = $mem$$index;

     int scale    = $mem$$scale;

     int displace = $mem$$disp;

     relocInfo::relocType disp_reloc = $mem->disp_reloc(); // disp-as-oop when working with static globals

     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_reloc);

   %}

   enc_class RegLea (rRegI dst, rRegI src0, immI src1 ) %{    // emit_reg_lea

     int reg_encoding = $dst$$reg;

     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base

     int index        = 0x04;            // 0x04 indicates no index

     int scale        = 0x00;            // 0x00 indicates no scale

     int displace     = $src1$$constant; // 0x00 indicates no displacement

     relocInfo::relocType disp_reloc = relocInfo::none;

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_reloc);

   %}

   enc_class min_enc (rRegI dst, rRegI src) %{    // MIN

     // Compare dst,src

     emit_opcode(cbuf,0x3B);

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

     // jmp dst < src around move

     emit_opcode(cbuf,0x7C);

     emit_d8(cbuf,2);

     // move dst,src

     emit_opcode(cbuf,0x8B);

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class max_enc (rRegI dst, rRegI src) %{    // MAX

     // Compare dst,src

     emit_opcode(cbuf,0x3B);

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

     // jmp dst > src around move

     emit_opcode(cbuf,0x7F);

     emit_d8(cbuf,2);

     // move dst,src

     emit_opcode(cbuf,0x8B);

     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);

   %}

   enc_class enc_FPR_store(memory mem, regDPR src) %{

     // If src is FPR1, we can just FST to store it.

     // Else we need to FLD it to FPR1, then FSTP to store/pop it.

     int reg_encoding = 0x2; // Just store

     int base  = $mem$$base;

     int index = $mem$$index;

     int scale = $mem$$scale;

     int displace = $mem$$disp;

     relocInfo::relocType disp_reloc = $mem->disp_reloc(); // disp-as-oop when working with static globals

     if( $src$$reg != FPR1L_enc ) {

       reg_encoding = 0x3;  // Store & pop

       emit_opcode( cbuf, 0xD9 ); // FLD (i.e., push it)

       emit_d8( cbuf, 0xC0-1+$src$$reg );

     }

     cbuf.set_insts_mark();       // Mark start of opcode for reloc info in mem operand

     emit_opcode(cbuf,$primary);

     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_reloc);

   %}

   enc_class neg_reg(rRegI dst) %{

     // NEG $dst

     emit_opcode(cbuf,0xF7);

     emit_rm(cbuf, 0x3, 0x03, $dst$$reg );

   %}

   enc_class setLT_reg(eCXRegI dst) %{

     // SETLT $dst

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0x9C);

     emit_rm( cbuf, 0x3, 0x4, $dst$$reg );

   %}

   enc_class enc_cmpLTP(ncxRegI p, ncxRegI q, ncxRegI y, eCXRegI tmp) %{    // cadd_cmpLT

     int tmpReg = $tmp$$reg;

     // SUB $p,$q

     emit_opcode(cbuf,0x2B);

     emit_rm(cbuf, 0x3, $p$$reg, $q$$reg);

     // SBB $tmp,$tmp

     emit_opcode(cbuf,0x1B);

     emit_rm(cbuf, 0x3, tmpReg, tmpReg);

     // AND $tmp,$y

     emit_opcode(cbuf,0x23);

     emit_rm(cbuf, 0x3, tmpReg, $y$$reg);

     // ADD $p,$tmp

     emit_opcode(cbuf,0x03);

     emit_rm(cbuf, 0x3, $p$$reg, tmpReg);

   %}

   enc_class shift_left_long( eRegL dst, eCXRegI shift ) %{

     // TEST shift,32

     emit_opcode(cbuf,0xF7);

     emit_rm(cbuf, 0x3, 0, ECX_enc);

     emit_d32(cbuf,0x20);

     // JEQ,s small

     emit_opcode(cbuf, 0x74);

     emit_d8(cbuf, 0x04);

     // MOV    $dst.hi,$dst.lo

     emit_opcode( cbuf, 0x8B );

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg );

     // CLR    $dst.lo

     emit_opcode(cbuf, 0x33);

     emit_rm(cbuf, 0x3, $dst$$reg, $dst$$reg);

 // small:

     // SHLD   $dst.hi,$dst.lo,$shift

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0xA5);

     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));

     // SHL    $dst.lo,$shift"

     emit_opcode(cbuf,0xD3);

     emit_rm(cbuf, 0x3, 0x4, $dst$$reg );

   %}

   enc_class shift_right_long( eRegL dst, eCXRegI shift ) %{

     // TEST shift,32

     emit_opcode(cbuf,0xF7);

     emit_rm(cbuf, 0x3, 0, ECX_enc);

     emit_d32(cbuf,0x20);

     // JEQ,s small

     emit_opcode(cbuf, 0x74);

     emit_d8(cbuf, 0x04);

     // MOV    $dst.lo,$dst.hi

     emit_opcode( cbuf, 0x8B );

     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );

     // CLR    $dst.hi

     emit_opcode(cbuf, 0x33);

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($dst$$reg));

 // small:

     // SHRD   $dst.lo,$dst.hi,$shift

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0xAD);

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);

     // SHR    $dst.hi,$shift"

     emit_opcode(cbuf,0xD3);

     emit_rm(cbuf, 0x3, 0x5, HIGH_FROM_LOW($dst$$reg) );

   %}

   enc_class shift_right_arith_long( eRegL dst, eCXRegI shift ) %{

     // TEST shift,32

     emit_opcode(cbuf,0xF7);

     emit_rm(cbuf, 0x3, 0, ECX_enc);

     emit_d32(cbuf,0x20);

     // JEQ,s small

     emit_opcode(cbuf, 0x74);

     emit_d8(cbuf, 0x05);

     // MOV    $dst.lo,$dst.hi

     emit_opcode( cbuf, 0x8B );

     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );

     // SAR    $dst.hi,31

     emit_opcode(cbuf, 0xC1);

     emit_rm(cbuf, 0x3, 7, HIGH_FROM_LOW($dst$$reg) );

     emit_d8(cbuf, 0x1F );

 // small:

     // SHRD   $dst.lo,$dst.hi,$shift

     emit_opcode(cbuf,0x0F);

     emit_opcode(cbuf,0xAD);

     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);

     // SAR    $dst.hi,$shift"

     emit_opcode(cbuf,0xD3);

     emit_rm(cbuf, 0x3, 0x7, HIGH_FROM_LOW($dst$$reg) );

   %}

   // ----------------- Encodings for floating point unit -----------------

   // May leave result in FPU-TOS or FPU reg depending on opcodes

   enc_class OpcReg_FPR(regFPR src) %{    // FMUL, FDIV

     $$$emit8$primary;

     emit_rm(cbuf, 0x3, $secondary, $src$$reg );

   %}

   // Pop argument in FPR0 with FSTP ST(0)

   enc_class PopFPU() %{

     emit_opcode( cbuf, 0xDD );

     emit_d8( cbuf, 0xD8 );

   %}

   // !!!!! equivalent to Pop_Reg_F

   enc_class Pop_Reg_DPR( regDPR dst ) %{

     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)

     emit_d8( cbuf, 0xD8+$dst$$reg );

   %}

   enc_class Push_Reg_DPR( regDPR dst ) %{

     emit_opcode( cbuf, 0xD9 );

     emit_d8( cbuf, 0xC0-1+$dst$$reg );   // FLD ST(i-1)

   %}

   enc_class strictfp_bias1( regDPR dst ) %{

     emit_opcode( cbuf, 0xDB );           // FLD m80real

     emit_opcode( cbuf, 0x2D );

     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias1() );

     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0

     emit_opcode( cbuf, 0xC8+$dst$$reg );

   %}

   enc_class strictfp_bias2( regDPR dst ) %{

     emit_opcode( cbuf, 0xDB );           // FLD m80real

     emit_opcode( cbuf, 0x2D );

     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias2() );

     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0

     emit_opcode( cbuf, 0xC8+$dst$$reg );

   %}

   // Special case for moving an integer register to a stack slot.

   enc_class OpcPRegSS( stackSlotI dst, rRegI src ) %{ // RegSS

     store_to_stackslot( cbuf, $primary, $src$$reg, $dst$$disp );

   %}

   // Special case for moving a register to a stack slot.

   enc_class RegSS( stackSlotI dst, rRegI src ) %{ // RegSS

     // Opcode already emitted

     emit_rm( cbuf, 0x02, $src$$reg, ESP_enc );   // R/M byte

     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);          // SIB byte

     emit_d32(cbuf, $dst$$disp);   // Displacement

   %}

   // Push the integer in stackSlot 'src' onto FP-stack

   enc_class Push_Mem_I( memory src ) %{    // FILD   [ESP+src]

     store_to_stackslot( cbuf, $primary, $secondary, $src$$disp );

   %}

   // Push FPU's TOS float to a stack-slot, and pop FPU-stack

   enc_class Pop_Mem_FPR( stackSlotF dst ) %{ // FSTP_S [ESP+dst]

     store_to_stackslot( cbuf, 0xD9, 0x03, $dst$$disp );

   %}

   // Same as Pop_Mem_F except for opcode

   // Push FPU's TOS double to a stack-slot, and pop FPU-stack

   enc_class Pop_Mem_DPR( stackSlotD dst ) %{ // FSTP_D [ESP+dst]

     store_to_stackslot( cbuf, 0xDD, 0x03, $dst$$disp );

   %}

   enc_class Pop_Reg_FPR( regFPR dst ) %{

     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)

     emit_d8( cbuf, 0xD8+$dst$$reg );

   %}

   enc_class Push_Reg_FPR( regFPR dst ) %{

     emit_opcode( cbuf, 0xD9 );           // FLD    ST(i-1)

     emit_d8( cbuf, 0xC0-1+$dst$$reg );

   %}

   // Push FPU's float to a stack-slot, and pop FPU-stack

   enc_class Pop_Mem_Reg_FPR( stackSlotF dst, regFPR src ) %{

     int pop = 0x02;

     if ($src$$reg != FPR1L_enc) {

       emit_opcode( cbuf, 0xD9 );         // FLD    ST(i-1)

       emit_d8( cbuf, 0xC0-1+$src$$reg );

       pop = 0x03;

     }

     store_to_stackslot( cbuf, 0xD9, pop, $dst$$disp ); // FST<P>_S  [ESP+dst]

   %}

   // Push FPU's double to a stack-slot, and pop FPU-stack

   enc_class Pop_Mem_Reg_DPR( stackSlotD dst, regDPR src ) %{

     int pop = 0x02;

     if ($src$$reg != FPR1L_enc) {

       emit_opcode( cbuf, 0xD9 );         // FLD    ST(i-1)

       emit_d8( cbuf, 0xC0-1+$src$$reg );

       pop = 0x03;

     }

     store_to_stackslot( cbuf, 0xDD, pop, $dst$$disp ); // FST<P>_D  [ESP+dst]

   %}

   // Push FPU's double to a FPU-stack-slot, and pop FPU-stack

   enc_class Pop_Reg_Reg_DPR( regDPR dst, regFPR src ) %{

     int pop = 0xD0 - 1; // -1 since we skip FLD

     if ($src$$reg != FPR1L_enc) {

       emit_opcode( cbuf, 0xD9 );         // FLD    ST(src-1)

       emit_d8( cbuf, 0xC0-1+$src$$reg );

       pop = 0xD8;

     }

     emit_opcode( cbuf, 0xDD );

     emit_d8( cbuf, pop+$dst$$reg );      // FST<P> ST(i)

   %}

   enc_class Push_Reg_Mod_DPR( regDPR dst, regDPR src) %{

     // load dst in FPR0

     emit_opcode( cbuf, 0xD9 );

     emit_d8( cbuf, 0xC0-1+$dst$$reg );

     if ($src$$reg != FPR1L_enc) {

       // fincstp

       emit_opcode (cbuf, 0xD9);

       emit_opcode (cbuf, 0xF7);

       // swap src with FPR1:

       // FXCH FPR1 with src

       emit_opcode(cbuf, 0xD9);

       emit_d8(cbuf, 0xC8-1+$src$$reg );

       // fdecstp

       emit_opcode (cbuf, 0xD9);

       emit_opcode (cbuf, 0xF6);

     }

   %}

   enc_class Push_ModD_encoding(regD src0, regD src1) %{

     MacroAssembler _masm(&cbuf);

     __ subptr(rsp, 8);

     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);

     __ fld_d(Address(rsp, 0));

     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);

     __ fld_d(Address(rsp, 0));

   %}

   enc_class Push_ModF_encoding(regF src0, regF src1) %{

     MacroAssembler _masm(&cbuf);

     __ subptr(rsp, 4);

     __ movflt(Address(rsp, 0), $src1$$XMMRegister);

     __ fld_s(Address(rsp, 0));

     __ movflt(Address(rsp, 0), $src0$$XMMRegister);

     __ fld_s(Address(rsp, 0));

   %}

   enc_class Push_ResultD(regD dst) %{

     MacroAssembler _masm(&cbuf);

     __ fstp_d(Address(rsp, 0));

     __ movdbl($dst$$XMMRegister, Address(rsp, 0));

     __ addptr(rsp, 8);

   %}

   enc_class Push_ResultF(regF dst, immI d8) %{

     MacroAssembler _masm(&cbuf);

     __ fstp_s(Address(rsp, 0));

     __ movflt($dst$$XMMRegister, Address(rsp, 0));

     __ addptr(rsp, $d8$$constant);

   %}

   enc_class Push_SrcD(regD src) %{

     MacroAssembler _masm(&cbuf);

     __ subptr(rsp, 8);

     __ movdbl(Address(rsp, 0), $src$$XMMRegister);

     __ fld_d(Address(rsp, 0));

   %}

   enc_class push_stack_temp_qword() %{

     MacroAssembler _masm(&cbuf);

     __ subptr(rsp, 8);

   %}

   enc_class pop_stack_temp_qword() %{

     MacroAssembler _masm(&cbuf);

     __ addptr(rsp, 8);

   %}

   enc_class push_xmm_to_fpr1(regD src) %{

     MacroAssembler _masm(&cbuf);

     __ movdbl(Address(rsp, 0), $src$$XMMRegister);

     __ fld_d(Address(rsp, 0));

   %}

   enc_class Push_Result_Mod_DPR( regDPR src) %{

     if ($src$$reg != FPR1L_enc) {

       // fincstp

       emit_opcode (cbuf, 0xD9);

       emit_opcode (cbuf, 0xF7);

       // FXCH FPR1 with src

       emit_opcode(cbuf, 0xD9);

       emit_d8(cbuf, 0xC8-1+$src$$reg );

       // fdecstp

       emit_opcode (cbuf, 0xD9);

       emit_opcode (cbuf, 0xF6);

     }

     // // following asm replaced with Pop_Reg_F or Pop_Mem_F

     // // FSTP   FPR$dst$$reg

     // emit_opcode( cbuf, 0xDD );

     // emit_d8( cbuf, 0xD8+$dst$$reg );

   %}

   enc_class fnstsw_sahf_skip_parity() %{

     // fnstsw ax

     emit_opcode( cbuf, 0xDF );

     emit_opcode( cbuf, 0xE0 );

     // sahf

     emit_opcode( cbuf, 0x9E );

     // jnp  ::skip

     emit_opcode( cbuf, 0x7B );

     emit_opcode( cbuf, 0x05 );

   %}

   enc_class emitModDPR() %{

     // fprem must be iterative

     // :: loop

     // fprem

     emit_opcode( cbuf, 0xD9 );

     emit_opcode( cbuf, 0xF8 );

     // wait

     emit_opcode( cbuf, 0x9b );

     // fnstsw ax

     emit_opcode( cbuf, 0xDF );

     emit_opcode( cbuf, 0xE0 );

     // sahf

     emit_opcode( cbuf, 0x9E );

     // jp  ::loop

     emit_opcode( cbuf, 0x0F );

     emit_opcode( cbuf, 0x8A );

     emit_opcode( cbuf, 0xF4 );

     emit_opcode( cbuf, 0xFF );

     emit_opcode( cbuf, 0xFF );

     emit_opcode( cbuf, 0xFF );

   %}

   enc_class fpu_flags() %{

     // fnstsw_ax

     emit_opcode( cbuf, 0xDF);

     emit_opcode( cbuf, 0xE0);

     // test ax,0x0400

     emit_opcode( cbuf, 0x66 );   // operand-size prefix for 16-bit immediate

     emit_opcode( cbuf, 0xA9 );

     emit_d16   ( cbuf, 0x0400 );

     // // // This sequence works, but stalls for 12-16 cycles on PPro

     // // test rax,0x0400

     // emit_opcode( cbuf, 0xA9 );

     // emit_d32   ( cbuf, 0x00000400 );

     //

     // jz exit (no unordered comparison)

     emit_opcode( cbuf, 0x74 );

     emit_d8    ( cbuf, 0x02 );

     // mov ah,1 - treat as LT case (set carry flag)

     emit_opcode( cbuf, 0xB4 );

     emit_d8    ( cbuf, 0x01 );

     // sahf

     emit_opcode( cbuf, 0x9E);

   %}

   enc_class cmpF_P6_fixup() %{

     // Fixup the integer flags in case comparison involved a NaN

     //

     // JNP exit (no unordered comparison, P-flag is set by NaN)

     emit_opcode( cbuf, 0x7B );

     emit_d8    ( cbuf, 0x03 );