Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "precompiled.hpp"

 #include "assembler_x86.inline.hpp"

 #include "gc_interface/collectedHeap.inline.hpp"

 #include "interpreter/interpreter.hpp"

 #include "memory/cardTableModRefBS.hpp"

 #include "memory/resourceArea.hpp"

 #include "prims/methodHandles.hpp"

 #include "runtime/biasedLocking.hpp"

 #include "runtime/interfaceSupport.hpp"

 #include "runtime/objectMonitor.hpp"

 #include "runtime/os.hpp"

 #include "runtime/sharedRuntime.hpp"

 #include "runtime/stubRoutines.hpp"

 #ifndef SERIALGC

 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"

 #include "gc_implementation/g1/heapRegion.hpp"

 #endif

 // Implementation of AddressLiteral

 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {

   _is_lval = false;

   _target = target;

   switch (rtype) {

   case relocInfo::oop_type:

     // Oops are a special case. Normally they would be their own section

     // but in cases like icBuffer they are literals in the code stream that

     // we don't have a section for. We use none so that we get a literal address

     // which is always patchable.

     break;

   case relocInfo::external_word_type:

     _rspec = external_word_Relocation::spec(target);

     break;

   case relocInfo::internal_word_type:

     _rspec = internal_word_Relocation::spec(target);

     break;

   case relocInfo::opt_virtual_call_type:

     _rspec = opt_virtual_call_Relocation::spec();

     break;

   case relocInfo::static_call_type:

     _rspec = static_call_Relocation::spec();

     break;

   case relocInfo::runtime_call_type:

     _rspec = runtime_call_Relocation::spec();

     break;

   case relocInfo::poll_type:

   case relocInfo::poll_return_type:

     _rspec = Relocation::spec_simple(rtype);

     break;

   case relocInfo::none:

     break;

   default:

     ShouldNotReachHere();

     break;

   }

 }

 // Implementation of Address

 #ifdef _LP64

 Address Address::make_array(ArrayAddress adr) {

   // Not implementable on 64bit machines

   // Should have been handled higher up the call chain.

   ShouldNotReachHere();

   return Address();

 }

 // exceedingly dangerous constructor

 Address::Address(int disp, address loc, relocInfo::relocType rtype) {

   _base  = noreg;

   _index = noreg;

   _scale = no_scale;

   _disp  = disp;

   switch (rtype) {

     case relocInfo::external_word_type:

       _rspec = external_word_Relocation::spec(loc);

       break;

     case relocInfo::internal_word_type:

       _rspec = internal_word_Relocation::spec(loc);

       break;

     case relocInfo::runtime_call_type:

       // HMM

       _rspec = runtime_call_Relocation::spec();

       break;

     case relocInfo::poll_type:

     case relocInfo::poll_return_type:

       _rspec = Relocation::spec_simple(rtype);

       break;

     case relocInfo::none:

       break;

     default:

       ShouldNotReachHere();

   }

 }

 #else // LP64

 Address Address::make_array(ArrayAddress adr) {

   AddressLiteral base = adr.base();

   Address index = adr.index();

   assert(index._disp == 0, "must not have disp"); // maybe it can?

   Address array(index._base, index._index, index._scale, (intptr_t) base.target());

   array._rspec = base._rspec;

   return array;

 }

 // exceedingly dangerous constructor

 Address::Address(address loc, RelocationHolder spec) {

   _base  = noreg;

   _index = noreg;

   _scale = no_scale;

   _disp  = (intptr_t) loc;

   _rspec = spec;

 }

 #endif // _LP64

 // Convert the raw encoding form into the form expected by the constructor for

 // Address.  An index of 4 (rsp) corresponds to having no index, so convert

 // that to noreg for the Address constructor.

 Address Address::make_raw(int base, int index, int scale, int disp, bool disp_is_oop) {

   RelocationHolder rspec;

   if (disp_is_oop) {

     rspec = Relocation::spec_simple(relocInfo::oop_type);

   }

   bool valid_index = index != rsp->encoding();

   if (valid_index) {

     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));

     madr._rspec = rspec;

     return madr;

   } else {

     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));

     madr._rspec = rspec;

     return madr;

   }

 }

 // Implementation of Assembler

 int AbstractAssembler::code_fill_byte() {

   return (u_char)'\xF4'; // hlt

 }

 // make this go away someday

 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {

   if (rtype == relocInfo::none)

         emit_long(data);

   else  emit_data(data, Relocation::spec_simple(rtype), format);

 }

 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {

   assert(imm_operand == 0, "default format must be immediate in this file");

   assert(inst_mark() != NULL, "must be inside InstructionMark");

   if (rspec.type() !=  relocInfo::none) {

     #ifdef ASSERT

       check_relocation(rspec, format);

     #endif

     // Do not use AbstractAssembler::relocate, which is not intended for

     // embedded words.  Instead, relocate to the enclosing instruction.

     // hack. call32 is too wide for mask so use disp32

     if (format == call32_operand)

       code_section()->relocate(inst_mark(), rspec, disp32_operand);

     else

       code_section()->relocate(inst_mark(), rspec, format);

   }

   emit_long(data);

 }

 static int encode(Register r) {

   int enc = r->encoding();

   if (enc >= 8) {

     enc -= 8;

   }

   return enc;

 }

 static int encode(XMMRegister r) {

   int enc = r->encoding();

   if (enc >= 8) {

     enc -= 8;

   }

   return enc;

 }

 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {

   assert(dst->has_byte_register(), "must have byte register");

   assert(isByte(op1) && isByte(op2), "wrong opcode");

   assert(isByte(imm8), "not a byte");

   assert((op1 & 0x01) == 0, "should be 8bit operation");

   emit_byte(op1);

   emit_byte(op2 | encode(dst));

   emit_byte(imm8);

 }

 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {

   assert(isByte(op1) && isByte(op2), "wrong opcode");

   assert((op1 & 0x01) == 1, "should be 32bit operation");

   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");

   if (is8bit(imm32)) {

     emit_byte(op1 | 0x02); // set sign bit

     emit_byte(op2 | encode(dst));

     emit_byte(imm32 & 0xFF);

   } else {

     emit_byte(op1);

     emit_byte(op2 | encode(dst));

     emit_long(imm32);

   }

 }

 // immediate-to-memory forms

 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {

   assert((op1 & 0x01) == 1, "should be 32bit operation");

   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");

   if (is8bit(imm32)) {

     emit_byte(op1 | 0x02); // set sign bit

     emit_operand(rm, adr, 1);

     emit_byte(imm32 & 0xFF);

   } else {

     emit_byte(op1);

     emit_operand(rm, adr, 4);

     emit_long(imm32);

   }

 }

 void Assembler::emit_arith(int op1, int op2, Register dst, jobject obj) {

   LP64_ONLY(ShouldNotReachHere());

   assert(isByte(op1) && isByte(op2), "wrong opcode");

   assert((op1 & 0x01) == 1, "should be 32bit operation");

   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");

   InstructionMark im(this);

   emit_byte(op1);

   emit_byte(op2 | encode(dst));

   emit_data((intptr_t)obj, relocInfo::oop_type, 0);

 }

 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {

   assert(isByte(op1) && isByte(op2), "wrong opcode");

   emit_byte(op1);

   emit_byte(op2 | encode(dst) << 3 | encode(src));

 }

 void Assembler::emit_operand(Register reg, Register base, Register index,

                              Address::ScaleFactor scale, int disp,

                              RelocationHolder const& rspec,

                              int rip_relative_correction) {

   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();

   // Encode the registers as needed in the fields they are used in

   int regenc = encode(reg) << 3;

   int indexenc = index->is_valid() ? encode(index) << 3 : 0;

   int baseenc = base->is_valid() ? encode(base) : 0;

   if (base->is_valid()) {

     if (index->is_valid()) {

       assert(scale != Address::no_scale, "inconsistent address");

       // [base + index*scale + disp]

       if (disp == 0 && rtype == relocInfo::none  &&

           base != rbp LP64_ONLY(&& base != r13)) {

         // [base + index*scale]

         // [00 reg 100][ss index base]

         assert(index != rsp, "illegal addressing mode");

         emit_byte(0x04 | regenc);

         emit_byte(scale << 6 | indexenc | baseenc);

       } else if (is8bit(disp) && rtype == relocInfo::none) {

         // [base + index*scale + imm8]

         // [01 reg 100][ss index base] imm8

         assert(index != rsp, "illegal addressing mode");

         emit_byte(0x44 | regenc);

         emit_byte(scale << 6 | indexenc | baseenc);

         emit_byte(disp & 0xFF);

       } else {

         // [base + index*scale + disp32]

         // [10 reg 100][ss index base] disp32

         assert(index != rsp, "illegal addressing mode");

         emit_byte(0x84 | regenc);

         emit_byte(scale << 6 | indexenc | baseenc);

         emit_data(disp, rspec, disp32_operand);

       }

     } else if (base == rsp LP64_ONLY(|| base == r12)) {

       // [rsp + disp]

       if (disp == 0 && rtype == relocInfo::none) {

         // [rsp]

         // [00 reg 100][00 100 100]

         emit_byte(0x04 | regenc);

         emit_byte(0x24);

       } else if (is8bit(disp) && rtype == relocInfo::none) {

         // [rsp + imm8]

         // [01 reg 100][00 100 100] disp8

         emit_byte(0x44 | regenc);

         emit_byte(0x24);

         emit_byte(disp & 0xFF);

       } else {

         // [rsp + imm32]

         // [10 reg 100][00 100 100] disp32

         emit_byte(0x84 | regenc);

         emit_byte(0x24);

         emit_data(disp, rspec, disp32_operand);

       }

     } else {

       // [base + disp]

       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");

       if (disp == 0 && rtype == relocInfo::none &&

           base != rbp LP64_ONLY(&& base != r13)) {

         // [base]

         // [00 reg base]

         emit_byte(0x00 | regenc | baseenc);

       } else if (is8bit(disp) && rtype == relocInfo::none) {

         // [base + disp8]

         // [01 reg base] disp8

         emit_byte(0x40 | regenc | baseenc);

         emit_byte(disp & 0xFF);

       } else {

         // [base + disp32]

         // [10 reg base] disp32

         emit_byte(0x80 | regenc | baseenc);

         emit_data(disp, rspec, disp32_operand);

       }

     }

   } else {

     if (index->is_valid()) {

       assert(scale != Address::no_scale, "inconsistent address");

       // [index*scale + disp]

       // [00 reg 100][ss index 101] disp32

       assert(index != rsp, "illegal addressing mode");

       emit_byte(0x04 | regenc);

       emit_byte(scale << 6 | indexenc | 0x05);

       emit_data(disp, rspec, disp32_operand);

     } else if (rtype != relocInfo::none ) {

       // [disp] (64bit) RIP-RELATIVE (32bit) abs

       // [00 000 101] disp32

       emit_byte(0x05 | regenc);

       // Note that the RIP-rel. correction applies to the generated

       // disp field, but _not_ to the target address in the rspec.

       // disp was created by converting the target address minus the pc

       // at the start of the instruction. That needs more correction here.

       // intptr_t disp = target - next_ip;

       assert(inst_mark() != NULL, "must be inside InstructionMark");

       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;

       int64_t adjusted = disp;

       // Do rip-rel adjustment for 64bit

       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));

       assert(is_simm32(adjusted),

              "must be 32bit offset (RIP relative address)");

       emit_data((int32_t) adjusted, rspec, disp32_operand);

     } else {

       // 32bit never did this, did everything as the rip-rel/disp code above

       // [disp] ABSOLUTE

       // [00 reg 100][00 100 101] disp32

       emit_byte(0x04 | regenc);

       emit_byte(0x25);

       emit_data(disp, rspec, disp32_operand);

     }

   }

 }

 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,

                              Address::ScaleFactor scale, int disp,

                              RelocationHolder const& rspec) {

   emit_operand((Register)reg, base, index, scale, disp, rspec);

 }

 // Secret local extension to Assembler::WhichOperand:

 #define end_pc_operand (_WhichOperand_limit)

 address Assembler::locate_operand(address inst, WhichOperand which) {

   // Decode the given instruction, and return the address of

   // an embedded 32-bit operand word.

   // If "which" is disp32_operand, selects the displacement portion

   // of an effective address specifier.

   // If "which" is imm64_operand, selects the trailing immediate constant.

   // If "which" is call32_operand, selects the displacement of a call or jump.

   // Caller is responsible for ensuring that there is such an operand,

   // and that it is 32/64 bits wide.

   // If "which" is end_pc_operand, find the end of the instruction.

   address ip = inst;

   bool is_64bit = false;

   debug_only(bool has_disp32 = false);

   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn

   again_after_prefix:

   switch (0xFF & *ip++) {

   // These convenience macros generate groups of "case" labels for the switch.

 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3

 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \

              case (x)+4: case (x)+5: case (x)+6: case (x)+7

 #define REP16(x) REP8((x)+0): \

               case REP8((x)+8)

   case CS_segment:

   case SS_segment:

   case DS_segment:

   case ES_segment:

   case FS_segment:

   case GS_segment:

     // Seems dubious

     LP64_ONLY(assert(false, "shouldn't have that prefix"));

     assert(ip == inst+1, "only one prefix allowed");

     goto again_after_prefix;

   case 0x67:

   case REX:

   case REX_B:

   case REX_X:

   case REX_XB:

   case REX_R:

   case REX_RB:

   case REX_RX:

   case REX_RXB:

     NOT_LP64(assert(false, "64bit prefixes"));

     goto again_after_prefix;

   case REX_W:

   case REX_WB:

   case REX_WX:

   case REX_WXB:

   case REX_WR:

   case REX_WRB:

   case REX_WRX:

   case REX_WRXB:

     NOT_LP64(assert(false, "64bit prefixes"));

     is_64bit = true;

     goto again_after_prefix;

   case 0xFF: // pushq a; decl a; incl a; call a; jmp a

   case 0x88: // movb a, r

   case 0x89: // movl a, r

   case 0x8A: // movb r, a

   case 0x8B: // movl r, a

   case 0x8F: // popl a

     debug_only(has_disp32 = true);

     break;

   case 0x68: // pushq #32

     if (which == end_pc_operand) {

       return ip + 4;

     }

     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");

     return ip;                  // not produced by emit_operand

   case 0x66: // movw ... (size prefix)

     again_after_size_prefix2:

     switch (0xFF & *ip++) {

     case REX:

     case REX_B:

     case REX_X:

     case REX_XB:

     case REX_R:

     case REX_RB:

     case REX_RX:

     case REX_RXB:

     case REX_W:

     case REX_WB:

     case REX_WX:

     case REX_WXB:

     case REX_WR:

     case REX_WRB:

     case REX_WRX:

     case REX_WRXB:

       NOT_LP64(assert(false, "64bit prefix found"));

       goto again_after_size_prefix2;

     case 0x8B: // movw r, a

     case 0x89: // movw a, r

       debug_only(has_disp32 = true);

       break;

     case 0xC7: // movw a, #16

       debug_only(has_disp32 = true);

       tail_size = 2;  // the imm16

       break;

     case 0x0F: // several SSE/SSE2 variants

       ip--;    // reparse the 0x0F

       goto again_after_prefix;

     default:

       ShouldNotReachHere();

     }

     break;

   case REP8(0xB8): // movl/q r, #32/#64(oop?)

     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);

     // these asserts are somewhat nonsensical

 #ifndef _LP64

     assert(which == imm_operand || which == disp32_operand, "");

 #else

     assert((which == call32_operand || which == imm_operand) && is_64bit ||

            which == narrow_oop_operand && !is_64bit, "");

 #endif // _LP64

     return ip;

   case 0x69: // imul r, a, #32

   case 0xC7: // movl a, #32(oop?)

     tail_size = 4;

     debug_only(has_disp32 = true); // has both kinds of operands!

     break;

   case 0x0F: // movx..., etc.

     switch (0xFF & *ip++) {

     case 0x3A: // pcmpestri

       tail_size = 1;

     case 0x38: // ptest, pmovzxbw

       ip++; // skip opcode

       debug_only(has_disp32 = true); // has both kinds of operands!

       break;

     case 0x70: // pshufd r, r/a, #8

       debug_only(has_disp32 = true); // has both kinds of operands!

     case 0x73: // psrldq r, #8

       tail_size = 1;

       break;

     case 0x12: // movlps

     case 0x28: // movaps

     case 0x2E: // ucomiss

     case 0x2F: // comiss

     case 0x54: // andps

     case 0x55: // andnps

     case 0x56: // orps

     case 0x57: // xorps

     case 0x6E: // movd

     case 0x7E: // movd

     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush

       debug_only(has_disp32 = true);

       break;

     case 0xAD: // shrd r, a, %cl

     case 0xAF: // imul r, a

     case 0xBE: // movsbl r, a (movsxb)

     case 0xBF: // movswl r, a (movsxw)

     case 0xB6: // movzbl r, a (movzxb)

     case 0xB7: // movzwl r, a (movzxw)

     case REP16(0x40): // cmovl cc, r, a

     case 0xB0: // cmpxchgb

     case 0xB1: // cmpxchg

     case 0xC1: // xaddl

     case 0xC7: // cmpxchg8

     case REP16(0x90): // setcc a

       debug_only(has_disp32 = true);

       // fall out of the switch to decode the address

       break;

     case 0xC4: // pinsrw r, a, #8

       debug_only(has_disp32 = true);

     case 0xC5: // pextrw r, r, #8

       tail_size = 1;  // the imm8

       break;

     case 0xAC: // shrd r, a, #8

       debug_only(has_disp32 = true);

       tail_size = 1;  // the imm8

       break;

     case REP16(0x80): // jcc rdisp32

       if (which == end_pc_operand)  return ip + 4;

       assert(which == call32_operand, "jcc has no disp32 or imm");

       return ip;

     default:

       ShouldNotReachHere();

     }

     break;

   case 0x81: // addl a, #32; addl r, #32

     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl

     // on 32bit in the case of cmpl, the imm might be an oop

     tail_size = 4;

     debug_only(has_disp32 = true); // has both kinds of operands!

     break;

   case 0x83: // addl a, #8; addl r, #8

     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl

     debug_only(has_disp32 = true); // has both kinds of operands!

     tail_size = 1;

     break;

   case 0x9B:

     switch (0xFF & *ip++) {

     case 0xD9: // fnstcw a

       debug_only(has_disp32 = true);

       break;

     default:

       ShouldNotReachHere();

     }

     break;

   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a

   case REP4(0x10): // adc...

   case REP4(0x20): // and...

   case REP4(0x30): // xor...

   case REP4(0x08): // or...

   case REP4(0x18): // sbb...

   case REP4(0x28): // sub...

   case 0xF7: // mull a

   case 0x8D: // lea r, a

   case 0x87: // xchg r, a

   case REP4(0x38): // cmp...

   case 0x85: // test r, a

     debug_only(has_disp32 = true); // has both kinds of operands!

     break;

   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8

   case 0xC6: // movb a, #8

   case 0x80: // cmpb a, #8

   case 0x6B: // imul r, a, #8

     debug_only(has_disp32 = true); // has both kinds of operands!

     tail_size = 1; // the imm8

     break;

   case 0xC4: // VEX_3bytes

   case 0xC5: // VEX_2bytes

     assert((UseAVX > 0), "shouldn't have VEX prefix");

     assert(ip == inst+1, "no prefixes allowed");

     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions

     // but they have prefix 0x0F and processed when 0x0F processed above.

     //

     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES

     // instructions (these instructions are not supported in 64-bit mode).

     // To distinguish them bits [7:6] are set in the VEX second byte since

     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set

     // those VEX bits REX and vvvv bits are inverted.

     //

     // Fortunately C2 doesn't generate these instructions so we don't need

     // to check for them in product version.

     // Check second byte

     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));

     // First byte

     if ((0xFF & *inst) == VEX_3bytes) {

       ip++; // third byte

       is_64bit = ((VEX_W & *ip) == VEX_W);

     }

     ip++; // opcode

     // To find the end of instruction (which == end_pc_operand).

     switch (0xFF & *ip) {

     case 0x61: // pcmpestri r, r/a, #8

     case 0x70: // pshufd r, r/a, #8

     case 0x73: // psrldq r, #8

       tail_size = 1;  // the imm8

       break;

     default:

       break;

     }

     ip++; // skip opcode

     debug_only(has_disp32 = true); // has both kinds of operands!

     break;

   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1

   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl

   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a

   case 0xDD: // fld_d a; fst_d a; fstp_d a

   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a

   case 0xDF: // fild_d a; fistp_d a

   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a

   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a

   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a

     debug_only(has_disp32 = true);

     break;

   case 0xE8: // call rdisp32

   case 0xE9: // jmp  rdisp32

     if (which == end_pc_operand)  return ip + 4;

     assert(which == call32_operand, "call has no disp32 or imm");

     return ip;

   case 0xF0:                    // Lock

     assert(os::is_MP(), "only on MP");

     goto again_after_prefix;

   case 0xF3:                    // For SSE

   case 0xF2:                    // For SSE2

     switch (0xFF & *ip++) {

     case REX:

     case REX_B:

     case REX_X:

     case REX_XB:

     case REX_R:

     case REX_RB:

     case REX_RX:

     case REX_RXB:

     case REX_W:

     case REX_WB:

     case REX_WX:

     case REX_WXB:

     case REX_WR:

     case REX_WRB:

     case REX_WRX:

     case REX_WRXB:

       NOT_LP64(assert(false, "found 64bit prefix"));

       ip++;

     default:

       ip++;

     }

     debug_only(has_disp32 = true); // has both kinds of operands!

     break;

   default:

     ShouldNotReachHere();

 #undef REP8

 #undef REP16

   }

   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");

 #ifdef _LP64

   assert(which != imm_operand, "instruction is not a movq reg, imm64");

 #else

   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");

   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");

 #endif // LP64

   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");

   // parse the output of emit_operand

   int op2 = 0xFF & *ip++;

   int base = op2 & 0x07;

   int op3 = -1;

   const int b100 = 4;

   const int b101 = 5;

   if (base == b100 && (op2 >> 6) != 3) {

     op3 = 0xFF & *ip++;

     base = op3 & 0x07;   // refetch the base

   }

   // now ip points at the disp (if any)

   switch (op2 >> 6) {

   case 0:

     // [00 reg  100][ss index base]

     // [00 reg  100][00   100  esp]

     // [00 reg base]

     // [00 reg  100][ss index  101][disp32]

     // [00 reg  101]               [disp32]

     if (base == b101) {

       if (which == disp32_operand)

         return ip;              // caller wants the disp32

       ip += 4;                  // skip the disp32

     }

     break;

   case 1:

     // [01 reg  100][ss index base][disp8]

     // [01 reg  100][00   100  esp][disp8]

     // [01 reg base]               [disp8]

     ip += 1;                    // skip the disp8

     break;

   case 2:

     // [10 reg  100][ss index base][disp32]

     // [10 reg  100][00   100  esp][disp32]

     // [10 reg base]               [disp32]

     if (which == disp32_operand)

       return ip;                // caller wants the disp32

     ip += 4;                    // skip the disp32

     break;

   case 3:

     // [11 reg base]  (not a memory addressing mode)

     break;

   }

   if (which == end_pc_operand) {

     return ip + tail_size;

   }

 #ifdef _LP64

   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");

 #else

   assert(which == imm_operand, "instruction has only an imm field");

 #endif // LP64

   return ip;

 }

 address Assembler::locate_next_instruction(address inst) {

   // Secretly share code with locate_operand:

   return locate_operand(inst, end_pc_operand);

 }

 #ifdef ASSERT

 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {

   address inst = inst_mark();

   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");

   address opnd;

   Relocation* r = rspec.reloc();

   if (r->type() == relocInfo::none) {

     return;

   } else if (r->is_call() || format == call32_operand) {

     // assert(format == imm32_operand, "cannot specify a nonzero format");

     opnd = locate_operand(inst, call32_operand);

   } else if (r->is_data()) {

     assert(format == imm_operand || format == disp32_operand

            LP64_ONLY(|| format == narrow_oop_operand), "format ok");

     opnd = locate_operand(inst, (WhichOperand)format);

   } else {

     assert(format == imm_operand, "cannot specify a format");

     return;

   }

   assert(opnd == pc(), "must put operand where relocs can find it");

 }

 #endif // ASSERT

 void Assembler::emit_operand32(Register reg, Address adr) {

   assert(reg->encoding() < 8, "no extended registers");

   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");

   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,

                adr._rspec);

 }

 void Assembler::emit_operand(Register reg, Address adr,

                              int rip_relative_correction) {

   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,

                adr._rspec,

                rip_relative_correction);

 }

 void Assembler::emit_operand(XMMRegister reg, Address adr) {

   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,

                adr._rspec);

 }

 // MMX operations

 void Assembler::emit_operand(MMXRegister reg, Address adr) {

   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");

   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);

 }

 // work around gcc (3.2.1-7a) bug

 void Assembler::emit_operand(Address adr, MMXRegister reg) {

   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");

   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);

 }

 void Assembler::emit_farith(int b1, int b2, int i) {

   assert(isByte(b1) && isByte(b2), "wrong opcode");

   assert(0 <= i &&  i < 8, "illegal stack offset");

   emit_byte(b1);

   emit_byte(b2 + i);

 }

 // Now the Assembler instructions (identical for 32/64 bits)

 void Assembler::adcl(Address dst, int32_t imm32) {

   InstructionMark im(this);

   prefix(dst);

   emit_arith_operand(0x81, rdx, dst, imm32);

 }

 void Assembler::adcl(Address dst, Register src) {

   InstructionMark im(this);

   prefix(dst, src);

   emit_byte(0x11);

   emit_operand(src, dst);

 }

 void Assembler::adcl(Register dst, int32_t imm32) {

   prefix(dst);

   emit_arith(0x81, 0xD0, dst, imm32);

 }

 void Assembler::adcl(Register dst, Address src) {

   InstructionMark im(this);

   prefix(src, dst);

   emit_byte(0x13);

   emit_operand(dst, src);

 }

 void Assembler::adcl(Register dst, Register src) {

   (void) prefix_and_encode(dst->encoding(), src->encoding());

   emit_arith(0x13, 0xC0, dst, src);

 }

 void Assembler::addl(Address dst, int32_t imm32) {

   InstructionMark im(this);

   prefix(dst);

   emit_arith_operand(0x81, rax, dst, imm32);

 }

 void Assembler::addl(Address dst, Register src) {

   InstructionMark im(this);

   prefix(dst, src);

   emit_byte(0x01);

   emit_operand(src, dst);

 }

 void Assembler::addl(Register dst, int32_t imm32) {

   prefix(dst);

   emit_arith(0x81, 0xC0, dst, imm32);

 }

 void Assembler::addl(Register dst, Address src) {

   InstructionMark im(this);

   prefix(src, dst);

   emit_byte(0x03);

   emit_operand(dst, src);

 }

 void Assembler::addl(Register dst, Register src) {

   (void) prefix_and_encode(dst->encoding(), src->encoding());

   emit_arith(0x03, 0xC0, dst, src);

 }

 void Assembler::addr_nop_4() {

   // 4 bytes: NOP DWORD PTR [EAX+0]

   emit_byte(0x0F);

   emit_byte(0x1F);

   emit_byte(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);

   emit_byte(0);    // 8-bits offset (1 byte)

 }

 void Assembler::addr_nop_5() {

   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset

   emit_byte(0x0F);

   emit_byte(0x1F);

   emit_byte(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);

   emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);

   emit_byte(0);    // 8-bits offset (1 byte)

 }

 void Assembler::addr_nop_7() {

   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset

   emit_byte(0x0F);

   emit_byte(0x1F);

   emit_byte(0x80); // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);

   emit_long(0);    // 32-bits offset (4 bytes)

 }

 void Assembler::addr_nop_8() {

   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset

   emit_byte(0x0F);

   emit_byte(0x1F);

   emit_byte(0x84); // emit_rm(cbuf, 0x2, EAX_enc, 0x4);

   emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);

   emit_long(0);    // 32-bits offset (4 bytes)

 }

 void Assembler::addsd(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x58);

   emit_byte(0xC0 | encode);

 }

 void Assembler::addsd(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x58);

   emit_operand(dst, src);

 }

 void Assembler::addss(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x58);

   emit_byte(0xC0 | encode);

 }

 void Assembler::addss(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x58);

   emit_operand(dst, src);

 }

 void Assembler::andl(Address dst, int32_t imm32) {

   InstructionMark im(this);

   prefix(dst);

   emit_byte(0x81);

   emit_operand(rsp, dst, 4);

   emit_long(imm32);

 }

 void Assembler::andl(Register dst, int32_t imm32) {

   prefix(dst);

   emit_arith(0x81, 0xE0, dst, imm32);

 }

 void Assembler::andl(Register dst, Address src) {

   InstructionMark im(this);

   prefix(src, dst);

   emit_byte(0x23);

   emit_operand(dst, src);

 }

 void Assembler::andl(Register dst, Register src) {

   (void) prefix_and_encode(dst->encoding(), src->encoding());

   emit_arith(0x23, 0xC0, dst, src);

 }

 void Assembler::andpd(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_66);

   emit_byte(0x54);

   emit_operand(dst, src);

 }

 void Assembler::andpd(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);

   emit_byte(0x54);

   emit_byte(0xC0 | encode);

 }

 void Assembler::andps(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_NONE);

   emit_byte(0x54);

   emit_operand(dst, src);

 }

 void Assembler::andps(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE);

   emit_byte(0x54);

   emit_byte(0xC0 | encode);

 }

 void Assembler::bsfl(Register dst, Register src) {

   int encode = prefix_and_encode(dst->encoding(), src->encoding());

   emit_byte(0x0F);

   emit_byte(0xBC);

   emit_byte(0xC0 | encode);

 }

 void Assembler::bsrl(Register dst, Register src) {

   assert(!VM_Version::supports_lzcnt(), "encoding is treated as LZCNT");

   int encode = prefix_and_encode(dst->encoding(), src->encoding());

   emit_byte(0x0F);

   emit_byte(0xBD);

   emit_byte(0xC0 | encode);

 }

 void Assembler::bswapl(Register reg) { // bswap

   int encode = prefix_and_encode(reg->encoding());

   emit_byte(0x0F);

   emit_byte(0xC8 | encode);

 }

 void Assembler::call(Label& L, relocInfo::relocType rtype) {

   // suspect disp32 is always good

   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);

   if (L.is_bound()) {

     const int long_size = 5;

     int offs = (int)( target(L) - pc() );

     assert(offs <= 0, "assembler error");

     InstructionMark im(this);

     // 1110 1000 #32-bit disp

     emit_byte(0xE8);

     emit_data(offs - long_size, rtype, operand);

   } else {

     InstructionMark im(this);

     // 1110 1000 #32-bit disp

     L.add_patch_at(code(), locator());

     emit_byte(0xE8);

     emit_data(int(0), rtype, operand);

   }

 }

 void Assembler::call(Register dst) {

   int encode = prefix_and_encode(dst->encoding());

   emit_byte(0xFF);

   emit_byte(0xD0 | encode);

 }

 void Assembler::call(Address adr) {

   InstructionMark im(this);

   prefix(adr);

   emit_byte(0xFF);

   emit_operand(rdx, adr);

 }

 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {

   assert(entry != NULL, "call most probably wrong");

   InstructionMark im(this);

   emit_byte(0xE8);

   intptr_t disp = entry - (_code_pos + sizeof(int32_t));

   assert(is_simm32(disp), "must be 32bit offset (call2)");

   // Technically, should use call32_operand, but this format is

   // implied by the fact that we're emitting a call instruction.

   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);

   emit_data((int) disp, rspec, operand);

 }

 void Assembler::cdql() {

   emit_byte(0x99);

 }

 void Assembler::cmovl(Condition cc, Register dst, Register src) {

   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));

   int encode = prefix_and_encode(dst->encoding(), src->encoding());

   emit_byte(0x0F);

   emit_byte(0x40 | cc);

   emit_byte(0xC0 | encode);

 }

 void Assembler::cmovl(Condition cc, Register dst, Address src) {

   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));

   prefix(src, dst);

   emit_byte(0x0F);

   emit_byte(0x40 | cc);

   emit_operand(dst, src);

 }

 void Assembler::cmpb(Address dst, int imm8) {

   InstructionMark im(this);

   prefix(dst);

   emit_byte(0x80);

   emit_operand(rdi, dst, 1);

   emit_byte(imm8);

 }

 void Assembler::cmpl(Address dst, int32_t imm32) {

   InstructionMark im(this);

   prefix(dst);

   emit_byte(0x81);

   emit_operand(rdi, dst, 4);

   emit_long(imm32);

 }

 void Assembler::cmpl(Register dst, int32_t imm32) {

   prefix(dst);

   emit_arith(0x81, 0xF8, dst, imm32);

 }

 void Assembler::cmpl(Register dst, Register src) {

   (void) prefix_and_encode(dst->encoding(), src->encoding());

   emit_arith(0x3B, 0xC0, dst, src);

 }

 void Assembler::cmpl(Register dst, Address  src) {

   InstructionMark im(this);

   prefix(src, dst);

   emit_byte(0x3B);

   emit_operand(dst, src);

 }

 void Assembler::cmpw(Address dst, int imm16) {

   InstructionMark im(this);

   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");

   emit_byte(0x66);

   emit_byte(0x81);

   emit_operand(rdi, dst, 2);

   emit_word(imm16);

 }

 // The 32-bit cmpxchg compares the value at adr with the contents of rax,

 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.

 // The ZF is set if the compared values were equal, and cleared otherwise.

 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg

   if (Atomics & 2) {

      // caveat: no instructionmark, so this isn't relocatable.

      // Emit a synthetic, non-atomic, CAS equivalent.

      // Beware.  The synthetic form sets all ICCs, not just ZF.

      // cmpxchg r,[m] is equivalent to rax, = CAS (m, rax, r)

      cmpl(rax, adr);

      movl(rax, adr);

      if (reg != rax) {

         Label L ;

         jcc(Assembler::notEqual, L);

         movl(adr, reg);

         bind(L);

      }

   } else {

      InstructionMark im(this);

      prefix(adr, reg);

      emit_byte(0x0F);

      emit_byte(0xB1);

      emit_operand(reg, adr);

   }

 }

 void Assembler::comisd(XMMRegister dst, Address src) {

   // NOTE: dbx seems to decode this as comiss even though the

   // 0x66 is there. Strangly ucomisd comes out correct

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_66);

   emit_byte(0x2F);

   emit_operand(dst, src);

 }

 void Assembler::comisd(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);

   emit_byte(0x2F);

   emit_byte(0xC0 | encode);

 }

 void Assembler::comiss(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_NONE);

   emit_byte(0x2F);

   emit_operand(dst, src);

 }

 void Assembler::comiss(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);

   emit_byte(0x2F);

   emit_byte(0xC0 | encode);

 }

 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);

   emit_byte(0xE6);

   emit_byte(0xC0 | encode);

 }

 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);

   emit_byte(0x5B);

   emit_byte(0xC0 | encode);

 }

 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x5A);

   emit_byte(0xC0 | encode);

 }

 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x5A);

   emit_operand(dst, src);

 }

 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x2A);

   emit_byte(0xC0 | encode);

 }

 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x2A);

   emit_operand(dst, src);

 }

 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x2A);

   emit_byte(0xC0 | encode);

 }

 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x2A);

   emit_operand(dst, src);

 }

 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x5A);

   emit_byte(0xC0 | encode);

 }

 void Assembler::cvtss2sd(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x5A);

   emit_operand(dst, src);

 }

 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2);

   emit_byte(0x2C);

   emit_byte(0xC0 | encode);

 }

 void Assembler::cvttss2sil(Register dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);

   emit_byte(0x2C);

   emit_byte(0xC0 | encode);

 }

 void Assembler::decl(Address dst) {

   // Don't use it directly. Use MacroAssembler::decrement() instead.

   InstructionMark im(this);

   prefix(dst);

   emit_byte(0xFF);

   emit_operand(rcx, dst);

 }

 void Assembler::divsd(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x5E);

   emit_operand(dst, src);

 }

 void Assembler::divsd(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x5E);

   emit_byte(0xC0 | encode);

 }

 void Assembler::divss(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x5E);

   emit_operand(dst, src);

 }

 void Assembler::divss(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x5E);

   emit_byte(0xC0 | encode);

 }

 void Assembler::emms() {

   NOT_LP64(assert(VM_Version::supports_mmx(), ""));

   emit_byte(0x0F);

   emit_byte(0x77);

 }

 void Assembler::hlt() {

   emit_byte(0xF4);

 }

 void Assembler::idivl(Register src) {

   int encode = prefix_and_encode(src->encoding());

   emit_byte(0xF7);

   emit_byte(0xF8 | encode);

 }

 void Assembler::divl(Register src) { // Unsigned

   int encode = prefix_and_encode(src->encoding());

   emit_byte(0xF7);

   emit_byte(0xF0 | encode);

 }

 void Assembler::imull(Register dst, Register src) {

   int encode = prefix_and_encode(dst->encoding(), src->encoding());

   emit_byte(0x0F);

   emit_byte(0xAF);

   emit_byte(0xC0 | encode);

 }

 void Assembler::imull(Register dst, Register src, int value) {

   int encode = prefix_and_encode(dst->encoding(), src->encoding());

   if (is8bit(value)) {

     emit_byte(0x6B);

     emit_byte(0xC0 | encode);

     emit_byte(value & 0xFF);

   } else {

     emit_byte(0x69);

     emit_byte(0xC0 | encode);

     emit_long(value);

   }

 }

 void Assembler::incl(Address dst) {

   // Don't use it directly. Use MacroAssembler::increment() instead.

   InstructionMark im(this);

   prefix(dst);

   emit_byte(0xFF);

   emit_operand(rax, dst);

 }

 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {

   InstructionMark im(this);

   assert((0 <= cc) && (cc < 16), "illegal cc");

   if (L.is_bound()) {

     address dst = target(L);

     assert(dst != NULL, "jcc most probably wrong");

     const int short_size = 2;

     const int long_size = 6;

     intptr_t offs = (intptr_t)dst - (intptr_t)_code_pos;

     if (maybe_short && is8bit(offs - short_size)) {

       // 0111 tttn #8-bit disp

       emit_byte(0x70 | cc);

       emit_byte((offs - short_size) & 0xFF);

     } else {

       // 0000 1111 1000 tttn #32-bit disp

       assert(is_simm32(offs - long_size),

              "must be 32bit offset (call4)");

       emit_byte(0x0F);

       emit_byte(0x80 | cc);

       emit_long(offs - long_size);

     }

   } else {

     // Note: could eliminate cond. jumps to this jump if condition

     //       is the same however, seems to be rather unlikely case.

     // Note: use jccb() if label to be bound is very close to get

     //       an 8-bit displacement

     L.add_patch_at(code(), locator());

     emit_byte(0x0F);

     emit_byte(0x80 | cc);

     emit_long(0);

   }

 }

 void Assembler::jccb(Condition cc, Label& L) {

   if (L.is_bound()) {

     const int short_size = 2;

     address entry = target(L);

     assert(is8bit((intptr_t)entry - ((intptr_t)_code_pos + short_size)),

            "Dispacement too large for a short jmp");

     intptr_t offs = (intptr_t)entry - (intptr_t)_code_pos;

     // 0111 tttn #8-bit disp

     emit_byte(0x70 | cc);

     emit_byte((offs - short_size) & 0xFF);

   } else {

     InstructionMark im(this);

     L.add_patch_at(code(), locator());

     emit_byte(0x70 | cc);

     emit_byte(0);

   }

 }

 void Assembler::jmp(Address adr) {

   InstructionMark im(this);

   prefix(adr);

   emit_byte(0xFF);

   emit_operand(rsp, adr);

 }

 void Assembler::jmp(Label& L, bool maybe_short) {

   if (L.is_bound()) {

     address entry = target(L);

     assert(entry != NULL, "jmp most probably wrong");

     InstructionMark im(this);

     const int short_size = 2;

     const int long_size = 5;

     intptr_t offs = entry - _code_pos;

     if (maybe_short && is8bit(offs - short_size)) {

       emit_byte(0xEB);

       emit_byte((offs - short_size) & 0xFF);

     } else {

       emit_byte(0xE9);

       emit_long(offs - long_size);

     }

   } else {

     // By default, forward jumps are always 32-bit displacements, since

     // we can't yet know where the label will be bound.  If you're sure that

     // the forward jump will not run beyond 256 bytes, use jmpb to

     // force an 8-bit displacement.

     InstructionMark im(this);

     L.add_patch_at(code(), locator());

     emit_byte(0xE9);

     emit_long(0);

   }

 }

 void Assembler::jmp(Register entry) {

   int encode = prefix_and_encode(entry->encoding());

   emit_byte(0xFF);

   emit_byte(0xE0 | encode);

 }

 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {

   InstructionMark im(this);

   emit_byte(0xE9);

   assert(dest != NULL, "must have a target");

   intptr_t disp = dest - (_code_pos + sizeof(int32_t));

   assert(is_simm32(disp), "must be 32bit offset (jmp)");

   emit_data(disp, rspec.reloc(), call32_operand);

 }

 void Assembler::jmpb(Label& L) {

   if (L.is_bound()) {

     const int short_size = 2;

     address entry = target(L);

     assert(is8bit((entry - _code_pos) + short_size),

            "Dispacement too large for a short jmp");

     assert(entry != NULL, "jmp most probably wrong");

     intptr_t offs = entry - _code_pos;

     emit_byte(0xEB);

     emit_byte((offs - short_size) & 0xFF);

   } else {

     InstructionMark im(this);

     L.add_patch_at(code(), locator());

     emit_byte(0xEB);

     emit_byte(0);

   }

 }

 void Assembler::ldmxcsr( Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   InstructionMark im(this);

   prefix(src);

   emit_byte(0x0F);

   emit_byte(0xAE);

   emit_operand(as_Register(2), src);

 }

 void Assembler::leal(Register dst, Address src) {

   InstructionMark im(this);

 #ifdef _LP64

   emit_byte(0x67); // addr32

   prefix(src, dst);

 #endif // LP64

   emit_byte(0x8D);

   emit_operand(dst, src);

 }

 void Assembler::lock() {

   if (Atomics & 1) {

      // Emit either nothing, a NOP, or a NOP: prefix

      emit_byte(0x90) ;

   } else {

      emit_byte(0xF0);

   }

 }

 void Assembler::lzcntl(Register dst, Register src) {

   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");

   emit_byte(0xF3);

   int encode = prefix_and_encode(dst->encoding(), src->encoding());

   emit_byte(0x0F);

   emit_byte(0xBD);

   emit_byte(0xC0 | encode);

 }

 // Emit mfence instruction

 void Assembler::mfence() {

   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)

   emit_byte( 0x0F );

   emit_byte( 0xAE );

   emit_byte( 0xF0 );

 }

 void Assembler::mov(Register dst, Register src) {

   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));

 }

 void Assembler::movapd(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);

   emit_byte(0x28);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movaps(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);

   emit_byte(0x28);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movb(Register dst, Address src) {

   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));

   InstructionMark im(this);

   prefix(src, dst, true);

   emit_byte(0x8A);

   emit_operand(dst, src);

 }

 void Assembler::movb(Address dst, int imm8) {

   InstructionMark im(this);

    prefix(dst);

   emit_byte(0xC6);

   emit_operand(rax, dst, 1);

   emit_byte(imm8);

 }

 void Assembler::movb(Address dst, Register src) {

   assert(src->has_byte_register(), "must have byte register");

   InstructionMark im(this);

   prefix(dst, src, true);

   emit_byte(0x88);

   emit_operand(src, dst);

 }

 void Assembler::movdl(XMMRegister dst, Register src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);

   emit_byte(0x6E);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movdl(Register dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   // swap src/dst to get correct prefix

   int encode = simd_prefix_and_encode(src, dst, VEX_SIMD_66);

   emit_byte(0x7E);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movdl(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_66);

   emit_byte(0x6E);

   emit_operand(dst, src);

 }

 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);

   emit_byte(0x6F);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movdqu(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_F3);

   emit_byte(0x6F);

   emit_operand(dst, src);

 }

 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);

   emit_byte(0x6F);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movdqu(Address dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_F3);

   emit_byte(0x7F);

   emit_operand(src, dst);

 }

 // Uses zero extension on 64bit

 void Assembler::movl(Register dst, int32_t imm32) {

   int encode = prefix_and_encode(dst->encoding());

   emit_byte(0xB8 | encode);

   emit_long(imm32);

 }

 void Assembler::movl(Register dst, Register src) {

   int encode = prefix_and_encode(dst->encoding(), src->encoding());

   emit_byte(0x8B);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movl(Register dst, Address src) {

   InstructionMark im(this);

   prefix(src, dst);

   emit_byte(0x8B);

   emit_operand(dst, src);

 }

 void Assembler::movl(Address dst, int32_t imm32) {

   InstructionMark im(this);

   prefix(dst);

   emit_byte(0xC7);

   emit_operand(rax, dst, 4);

   emit_long(imm32);

 }

 void Assembler::movl(Address dst, Register src) {

   InstructionMark im(this);

   prefix(dst, src);

   emit_byte(0x89);

   emit_operand(src, dst);

 }

 // New cpus require to use movsd and movss to avoid partial register stall

 // when loading from memory. But for old Opteron use movlpd instead of movsd.

 // The selection is done in MacroAssembler::movdbl() and movflt().

 void Assembler::movlpd(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_66);

   emit_byte(0x12);

   emit_operand(dst, src);

 }

 void Assembler::movq( MMXRegister dst, Address src ) {

   assert( VM_Version::supports_mmx(), "" );

   emit_byte(0x0F);

   emit_byte(0x6F);

   emit_operand(dst, src);

 }

 void Assembler::movq( Address dst, MMXRegister src ) {

   assert( VM_Version::supports_mmx(), "" );

   emit_byte(0x0F);

   emit_byte(0x7F);

   // workaround gcc (3.2.1-7a) bug

   // In that version of gcc with only an emit_operand(MMX, Address)

   // gcc will tail jump and try and reverse the parameters completely

   // obliterating dst in the process. By having a version available

   // that doesn't need to swap the args at the tail jump the bug is

   // avoided.

   emit_operand(dst, src);

 }

 void Assembler::movq(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_F3);

   emit_byte(0x7E);

   emit_operand(dst, src);

 }

 void Assembler::movq(Address dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_66);

   emit_byte(0xD6);

   emit_operand(src, dst);

 }

 void Assembler::movsbl(Register dst, Address src) { // movsxb

   InstructionMark im(this);

   prefix(src, dst);

   emit_byte(0x0F);

   emit_byte(0xBE);

   emit_operand(dst, src);

 }

 void Assembler::movsbl(Register dst, Register src) { // movsxb

   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));

   int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);

   emit_byte(0x0F);

   emit_byte(0xBE);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movsd(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x10);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movsd(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_F2);

   emit_byte(0x10);

   emit_operand(dst, src);

 }

 void Assembler::movsd(Address dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_F2);

   emit_byte(0x11);

   emit_operand(src, dst);

 }

 void Assembler::movss(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x10);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movss(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_F3);

   emit_byte(0x10);

   emit_operand(dst, src);

 }

 void Assembler::movss(Address dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_F3);

   emit_byte(0x11);

   emit_operand(src, dst);

 }

 void Assembler::movswl(Register dst, Address src) { // movsxw

   InstructionMark im(this);

   prefix(src, dst);

   emit_byte(0x0F);

   emit_byte(0xBF);

   emit_operand(dst, src);

 }

 void Assembler::movswl(Register dst, Register src) { // movsxw

   int encode = prefix_and_encode(dst->encoding(), src->encoding());

   emit_byte(0x0F);

   emit_byte(0xBF);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movw(Address dst, int imm16) {

   InstructionMark im(this);

   emit_byte(0x66); // switch to 16-bit mode

   prefix(dst);

   emit_byte(0xC7);

   emit_operand(rax, dst, 2);

   emit_word(imm16);

 }

 void Assembler::movw(Register dst, Address src) {

   InstructionMark im(this);

   emit_byte(0x66);

   prefix(src, dst);

   emit_byte(0x8B);

   emit_operand(dst, src);

 }

 void Assembler::movw(Address dst, Register src) {

   InstructionMark im(this);

   emit_byte(0x66);

   prefix(dst, src);

   emit_byte(0x89);

   emit_operand(src, dst);

 }

 void Assembler::movzbl(Register dst, Address src) { // movzxb

   InstructionMark im(this);

   prefix(src, dst);

   emit_byte(0x0F);

   emit_byte(0xB6);

   emit_operand(dst, src);

 }

 void Assembler::movzbl(Register dst, Register src) { // movzxb

   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));

   int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);

   emit_byte(0x0F);

   emit_byte(0xB6);

   emit_byte(0xC0 | encode);

 }

 void Assembler::movzwl(Register dst, Address src) { // movzxw

   InstructionMark im(this);

   prefix(src, dst);

   emit_byte(0x0F);

   emit_byte(0xB7);

   emit_operand(dst, src);

 }

 void Assembler::movzwl(Register dst, Register src) { // movzxw

   int encode = prefix_and_encode(dst->encoding(), src->encoding());

   emit_byte(0x0F);

   emit_byte(0xB7);

   emit_byte(0xC0 | encode);

 }

 void Assembler::mull(Address src) {

   InstructionMark im(this);

   prefix(src);

   emit_byte(0xF7);

   emit_operand(rsp, src);

 }

 void Assembler::mull(Register src) {

   int encode = prefix_and_encode(src->encoding());

   emit_byte(0xF7);

   emit_byte(0xE0 | encode);

 }

 void Assembler::mulsd(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x59);

   emit_operand(dst, src);

 }

 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);

   emit_byte(0x59);

   emit_byte(0xC0 | encode);

 }

 void Assembler::mulss(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x59);

   emit_operand(dst, src);

 }

 void Assembler::mulss(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);

   emit_byte(0x59);

   emit_byte(0xC0 | encode);

 }

 void Assembler::negl(Register dst) {

   int encode = prefix_and_encode(dst->encoding());

   emit_byte(0xF7);

   emit_byte(0xD8 | encode);

 }

 void Assembler::nop(int i) {

 #ifdef ASSERT

   assert(i > 0, " ");

   // The fancy nops aren't currently recognized by debuggers making it a

   // pain to disassemble code while debugging. If asserts are on clearly

   // speed is not an issue so simply use the single byte traditional nop

   // to do alignment.

   for (; i > 0 ; i--) emit_byte(0x90);

   return;

 #endif // ASSERT

   if (UseAddressNop && VM_Version::is_intel()) {

     //

     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel

     //  1: 0x90

     //  2: 0x66 0x90

     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)

     //  4: 0x0F 0x1F 0x40 0x00

     //  5: 0x0F 0x1F 0x44 0x00 0x00

     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00

     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00

     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00

     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00

     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00

     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00

     // The rest coding is Intel specific - don't use consecutive address nops

     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90

     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90

     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90

     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90

     while(i >= 15) {

       // For Intel don't generate consecutive addess nops (mix with regular nops)

       i -= 15;

       emit_byte(0x66);   // size prefix

       emit_byte(0x66);   // size prefix

       emit_byte(0x66);   // size prefix

       addr_nop_8();

       emit_byte(0x66);   // size prefix

       emit_byte(0x66);   // size prefix

       emit_byte(0x66);   // size prefix

       emit_byte(0x90);   // nop

     }

     switch (i) {

       case 14:

         emit_byte(0x66); // size prefix

       case 13:

         emit_byte(0x66); // size prefix

       case 12:

         addr_nop_8();

         emit_byte(0x66); // size prefix

         emit_byte(0x66); // size prefix

         emit_byte(0x66); // size prefix

         emit_byte(0x90); // nop

         break;

       case 11:

         emit_byte(0x66); // size prefix

       case 10:

         emit_byte(0x66); // size prefix

       case 9:

         emit_byte(0x66); // size prefix

       case 8:

         addr_nop_8();

         break;

       case 7:

         addr_nop_7();

         break;

       case 6:

         emit_byte(0x66); // size prefix

       case 5:

         addr_nop_5();

         break;

       case 4:

         addr_nop_4();

         break;

       case 3:

         // Don't use "0x0F 0x1F 0x00" - need patching safe padding

         emit_byte(0x66); // size prefix

       case 2:

         emit_byte(0x66); // size prefix

       case 1:

         emit_byte(0x90); // nop

         break;

       default:

         assert(i == 0, " ");

     }

     return;

   }

   if (UseAddressNop && VM_Version::is_amd()) {

     //

     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.

     //  1: 0x90

     //  2: 0x66 0x90

     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)

     //  4: 0x0F 0x1F 0x40 0x00

     //  5: 0x0F 0x1F 0x44 0x00 0x00

     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00

     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00

     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00

     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00

     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00

     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00

     // The rest coding is AMD specific - use consecutive address nops

     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00

     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00

     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00

     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00

     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00

     //     Size prefixes (0x66) are added for larger sizes

     while(i >= 22) {

       i -= 11;

       emit_byte(0x66); // size prefix

       emit_byte(0x66); // size prefix

       emit_byte(0x66); // size prefix

       addr_nop_8();

     }

     // Generate first nop for size between 21-12

     switch (i) {

       case 21:

         i -= 1;

         emit_byte(0x66); // size prefix

       case 20:

       case 19:

         i -= 1;

         emit_byte(0x66); // size prefix

       case 18:

       case 17:

         i -= 1;

         emit_byte(0x66); // size prefix

       case 16:

       case 15:

         i -= 8;

         addr_nop_8();

         break;

       case 14:

       case 13:

         i -= 7;

         addr_nop_7();

         break;

       case 12:

         i -= 6;

         emit_byte(0x66); // size prefix

         addr_nop_5();

         break;

       default:

         assert(i < 12, " ");

     }

     // Generate second nop for size between 11-1

     switch (i) {

       case 11:

         emit_byte(0x66); // size prefix

       case 10:

         emit_byte(0x66); // size prefix

       case 9:

         emit_byte(0x66); // size prefix

       case 8:

         addr_nop_8();

         break;

       case 7:

         addr_nop_7();

         break;

       case 6:

         emit_byte(0x66); // size prefix

       case 5:

         addr_nop_5();

         break;

       case 4:

         addr_nop_4();

         break;

       case 3:

         // Don't use "0x0F 0x1F 0x00" - need patching safe padding

         emit_byte(0x66); // size prefix

       case 2:

         emit_byte(0x66); // size prefix

       case 1:

         emit_byte(0x90); // nop

         break;

       default:

         assert(i == 0, " ");

     }

     return;

   }

   // Using nops with size prefixes "0x66 0x90".

   // From AMD Optimization Guide:

   //  1: 0x90

   //  2: 0x66 0x90

   //  3: 0x66 0x66 0x90

   //  4: 0x66 0x66 0x66 0x90

   //  5: 0x66 0x66 0x90 0x66 0x90

   //  6: 0x66 0x66 0x90 0x66 0x66 0x90

   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90

   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90

   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90

   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90

   //

   while(i > 12) {

     i -= 4;

     emit_byte(0x66); // size prefix

     emit_byte(0x66);

     emit_byte(0x66);

     emit_byte(0x90); // nop

   }

   // 1 - 12 nops

   if(i > 8) {

     if(i > 9) {

       i -= 1;

       emit_byte(0x66);

     }

     i -= 3;

     emit_byte(0x66);

     emit_byte(0x66);

     emit_byte(0x90);

   }

   // 1 - 8 nops

   if(i > 4) {

     if(i > 6) {

       i -= 1;

       emit_byte(0x66);

     }

     i -= 3;

     emit_byte(0x66);

     emit_byte(0x66);

     emit_byte(0x90);

   }

   switch (i) {

     case 4:

       emit_byte(0x66);

     case 3:

       emit_byte(0x66);

     case 2:

       emit_byte(0x66);

     case 1:

       emit_byte(0x90);

       break;

     default:

       assert(i == 0, " ");

   }

 }

 void Assembler::notl(Register dst) {

   int encode = prefix_and_encode(dst->encoding());

   emit_byte(0xF7);

   emit_byte(0xD0 | encode );

 }

 void Assembler::orl(Address dst, int32_t imm32) {

   InstructionMark im(this);

   prefix(dst);

   emit_arith_operand(0x81, rcx, dst, imm32);

 }

 void Assembler::orl(Register dst, int32_t imm32) {

   prefix(dst);

   emit_arith(0x81, 0xC8, dst, imm32);

 }

 void Assembler::orl(Register dst, Address src) {

   InstructionMark im(this);

   prefix(src, dst);

   emit_byte(0x0B);

   emit_operand(dst, src);

 }

 void Assembler::orl(Register dst, Register src) {

   (void) prefix_and_encode(dst->encoding(), src->encoding());

   emit_arith(0x0B, 0xC0, dst, src);

 }

 void Assembler::packuswb(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_66);

   emit_byte(0x67);

   emit_operand(dst, src);

 }

 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);

   emit_byte(0x67);

   emit_byte(0xC0 | encode);

 }

 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {

   assert(VM_Version::supports_sse4_2(), "");

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);

   emit_byte(0x61);

   emit_operand(dst, src);

   emit_byte(imm8);

 }

 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {

   assert(VM_Version::supports_sse4_2(), "");

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);

   emit_byte(0x61);

   emit_byte(0xC0 | encode);

   emit_byte(imm8);

 }

 void Assembler::pmovzxbw(XMMRegister dst, Address src) {

   assert(VM_Version::supports_sse4_1(), "");

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);

   emit_byte(0x30);

   emit_operand(dst, src);

 }

 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {

   assert(VM_Version::supports_sse4_1(), "");

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);

   emit_byte(0x30);

   emit_byte(0xC0 | encode);

 }

 // generic

 void Assembler::pop(Register dst) {

   int encode = prefix_and_encode(dst->encoding());

   emit_byte(0x58 | encode);

 }

 void Assembler::popcntl(Register dst, Address src) {

   assert(VM_Version::supports_popcnt(), "must support");

   InstructionMark im(this);

   emit_byte(0xF3);

   prefix(src, dst);

   emit_byte(0x0F);

   emit_byte(0xB8);

   emit_operand(dst, src);

 }

 void Assembler::popcntl(Register dst, Register src) {

   assert(VM_Version::supports_popcnt(), "must support");

   emit_byte(0xF3);

   int encode = prefix_and_encode(dst->encoding(), src->encoding());

   emit_byte(0x0F);

   emit_byte(0xB8);

   emit_byte(0xC0 | encode);

 }

 void Assembler::popf() {

   emit_byte(0x9D);

 }

 #ifndef _LP64 // no 32bit push/pop on amd64

 void Assembler::popl(Address dst) {

   // NOTE: this will adjust stack by 8byte on 64bits

   InstructionMark im(this);

   prefix(dst);

   emit_byte(0x8F);

   emit_operand(rax, dst);

 }

 #endif

 void Assembler::prefetch_prefix(Address src) {

   prefix(src);

   emit_byte(0x0F);

 }

 void Assembler::prefetchnta(Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));

   InstructionMark im(this);

   prefetch_prefix(src);

   emit_byte(0x18);

   emit_operand(rax, src); // 0, src

 }

 void Assembler::prefetchr(Address src) {

   assert(VM_Version::supports_3dnow_prefetch(), "must support");

   InstructionMark im(this);

   prefetch_prefix(src);

   emit_byte(0x0D);

   emit_operand(rax, src); // 0, src

 }

 void Assembler::prefetcht0(Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));

   InstructionMark im(this);

   prefetch_prefix(src);

   emit_byte(0x18);

   emit_operand(rcx, src); // 1, src

 }

 void Assembler::prefetcht1(Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));

   InstructionMark im(this);

   prefetch_prefix(src);

   emit_byte(0x18);

   emit_operand(rdx, src); // 2, src

 }

 void Assembler::prefetcht2(Address src) {

   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));

   InstructionMark im(this);

   prefetch_prefix(src);

   emit_byte(0x18);

   emit_operand(rbx, src); // 3, src

 }

 void Assembler::prefetchw(Address src) {

   assert(VM_Version::supports_3dnow_prefetch(), "must support");

   InstructionMark im(this);

   prefetch_prefix(src);

   emit_byte(0x0D);

   emit_operand(rcx, src); // 1, src

 }

 void Assembler::prefix(Prefix p) {

   a_byte(p);

 }

 void Assembler::por(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);

   emit_byte(0xEB);

   emit_byte(0xC0 | encode);

 }

 void Assembler::por(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_66);

   emit_byte(0xEB);

   emit_operand(dst, src);

 }

 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {

   assert(isByte(mode), "invalid value");

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);

   emit_byte(0x70);

   emit_byte(0xC0 | encode);

   emit_byte(mode & 0xFF);

 }

 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {

   assert(isByte(mode), "invalid value");

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_66);

   emit_byte(0x70);

   emit_operand(dst, src);

   emit_byte(mode & 0xFF);

 }

 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {

   assert(isByte(mode), "invalid value");

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2);

   emit_byte(0x70);

   emit_byte(0xC0 | encode);

   emit_byte(mode & 0xFF);

 }

 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {

   assert(isByte(mode), "invalid value");

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_F2);

   emit_byte(0x70);

   emit_operand(dst, src);

   emit_byte(mode & 0xFF);

 }

 void Assembler::psrlq(XMMRegister dst, int shift) {

   // Shift 64 bit value logically right by specified number of bits.

   // HMM Table D-1 says sse2 or mmx.

   // Do not confuse it with psrldq SSE2 instruction which

   // shifts 128 bit value in xmm register by number of bytes.

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);

   emit_byte(0x73);

   emit_byte(0xC0 | encode);

   emit_byte(shift);

 }

 void Assembler::psrldq(XMMRegister dst, int shift) {

   // Shift 128 bit value in xmm register by number of bytes.

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66);

   emit_byte(0x73);

   emit_byte(0xC0 | encode);

   emit_byte(shift);

 }

 void Assembler::ptest(XMMRegister dst, Address src) {

   assert(VM_Version::supports_sse4_1(), "");

   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");

   InstructionMark im(this);

   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);

   emit_byte(0x17);

   emit_operand(dst, src);

 }

 void Assembler::ptest(XMMRegister dst, XMMRegister src) {

   assert(VM_Version::supports_sse4_1(), "");

   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);

   emit_byte(0x17);

   emit_byte(0xC0 | encode);

 }

 void Assembler::punpcklbw(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_66);

   emit_byte(0x60);

   emit_operand(dst, src);

 }

 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);

   emit_byte(0x60);

   emit_byte(0xC0 | encode);

 }

 void Assembler::punpckldq(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_66);

   emit_byte(0x62);

   emit_operand(dst, src);

 }

 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);

   emit_byte(0x62);

   emit_byte(0xC0 | encode);

 }

 void Assembler::push(int32_t imm32) {

   // in 64bits we push 64bits onto the stack but only

   // take a 32bit immediate

   emit_byte(0x68);

   emit_long(imm32);

 }

 void Assembler::push(Register src) {

   int encode = prefix_and_encode(src->encoding());

   emit_byte(0x50 | encode);

 }

 void Assembler::pushf() {

   emit_byte(0x9C);

 }

 #ifndef _LP64 // no 32bit push/pop on amd64

 void Assembler::pushl(Address src) {

   // Note this will push 64bit on 64bit

   InstructionMark im(this);

   prefix(src);

   emit_byte(0xFF);

   emit_operand(rsi, src);

 }

 #endif

 void Assembler::pxor(XMMRegister dst, Address src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");

   InstructionMark im(this);

   simd_prefix(dst, dst, src, VEX_SIMD_66);

   emit_byte(0xEF);

   emit_operand(dst, src);

 }

 void Assembler::pxor(XMMRegister dst, XMMRegister src) {

   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);

   emit_byte(0xEF);

   emit_byte(0xC0 | encode);

 }

 void Assembler::rcll(Register dst, int imm8) {

   assert(isShiftCount(imm8), "illegal shift count");

   int encode = prefix_and_encode(dst->encoding());

   if (imm8 == 1) {

     emit_byte(0xD1);

     emit_byte(0xD0 | encode);

   } else {

     emit_byte(0xC1);

     emit_byte(0xD0 | encode);

     emit_byte(imm8);

   }

 }

 // copies data from [esi] to [edi] using rcx pointer sized words

 // generic

 void Assembler::rep_mov() {

   emit_byte(0xF3);

   // MOVSQ

   LP64_ONLY(prefix(REX_W));

   emit_byte(0xA5);

 }

 // sets rcx pointer sized words with rax, value at [edi]

 // generic

 void Assembler::rep_set() { // rep_set

   emit_byte(0xF3);

   // STOSQ

   LP64_ONLY(prefix(REX_W));

   emit_byte(0xAB);

 }

 // scans rcx pointer sized words at [edi] for occurance of rax,

 // generic

 void Assembler::repne_scan() { // repne_scan

   emit_byte(0xF2);

   // SCASQ

   LP64_ONLY(prefix(REX_W));

   emit_byte(0xAF);

 }

 #ifdef _LP64

 // scans rcx 4 byte words at [edi] for occurance of rax,

 // generic

 void Assembler::repne_scanl() { // repne_scan

   emit_byte(0xF2);

   // SCASL

   emit_byte(0xAF);

 }

 #endif

 void Assembler::ret(int imm16) {

   if (imm16 == 0) {

     emit_byte(0xC3);

   } else {

     emit_byte(0xC2);

     emit_word(imm16);

   }

 }

 void Assembler::sahf() {

 #ifdef _LP64

   // Not supported in 64bit mode

   ShouldNotReachHere();

 #endif

   emit_byte(0x9E);

 }

 void Assembler::sarl(Register dst, int imm8) {

   int encode = prefix_and_encode(dst->encoding());

   assert(isShiftCount(imm8), "illegal shift count");

   if (imm8 == 1) {

     emit_byte(0xD1);

     emit_byte(0xF8 | encode);

   } else {

     emit_byte(0xC1);

     emit_byte(0xF8 | encode);

     emit_byte(imm8);

   }

 }

 void Assembler::sarl(Register dst) {

   int encode = prefix_and_encode(dst->encoding());

   emit_byte(0xD3);

   emit_byte(0xF8 | encode);

 }

 void Assembler::sbbl(Address dst, int32_t imm32) {

   InstructionMark im(this);

   prefix(dst);

   emit_arith_operand(0x81, rbx, dst, imm32);

 }

 void Assembler::sbbl(Register dst, int32_t imm32) {

   prefix(dst);

   emit_arith(0x81, 0xD8, dst, imm32);

 }