Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "precompiled.hpp"

 #include "asm/assembler.hpp"

 #include "asm/assembler.inline.hpp"

 #include "compiler/disassembler.hpp"

 #include "gc_interface/collectedHeap.inline.hpp"

 #include "interpreter/interpreter.hpp"

 #include "memory/cardTableModRefBS.hpp"

 #include "memory/resourceArea.hpp"

 #include "prims/methodHandles.hpp"

 #include "runtime/biasedLocking.hpp"

 #include "runtime/interfaceSupport.hpp"

 #include "runtime/objectMonitor.hpp"

 #include "runtime/os.hpp"

 #include "runtime/sharedRuntime.hpp"

 #include "runtime/stubRoutines.hpp"

 #ifndef SERIALGC

 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"

 #include "gc_implementation/g1/heapRegion.hpp"

 #endif

 #ifdef PRODUCT

 #define BLOCK_COMMENT(str) /* nothing */

 #define STOP(error) stop(error)

 #else

 #define BLOCK_COMMENT(str) block_comment(str)

 #define STOP(error) block_comment(error); stop(error)

 #endif

 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

 #ifdef ASSERT

 bool AbstractAssembler::pd_check_instruction_mark() { return true; }

 #endif

 static Assembler::Condition reverse[] = {

     Assembler::noOverflow     /* overflow      = 0x0 */ ,

     Assembler::overflow       /* noOverflow    = 0x1 */ ,

     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,

     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,

     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,

     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,

     Assembler::above          /* belowEqual    = 0x6 */ ,

     Assembler::belowEqual     /* above         = 0x7 */ ,

     Assembler::positive       /* negative      = 0x8 */ ,

     Assembler::negative       /* positive      = 0x9 */ ,

     Assembler::noParity       /* parity        = 0xa */ ,

     Assembler::parity         /* noParity      = 0xb */ ,

     Assembler::greaterEqual   /* less          = 0xc */ ,

     Assembler::less           /* greaterEqual  = 0xd */ ,

     Assembler::greater        /* lessEqual     = 0xe */ ,

     Assembler::lessEqual      /* greater       = 0xf, */

 };

 // Implementation of MacroAssembler

 // First all the versions that have distinct versions depending on 32/64 bit

 // Unless the difference is trivial (1 line or so).

 #ifndef _LP64

 // 32bit versions

 Address MacroAssembler::as_Address(AddressLiteral adr) {

   return Address(adr.target(), adr.rspec());

 }

 Address MacroAssembler::as_Address(ArrayAddress adr) {

   return Address::make_array(adr);

 }

 int MacroAssembler::biased_locking_enter(Register lock_reg,

                                          Register obj_reg,

                                          Register swap_reg,

                                          Register tmp_reg,

                                          bool swap_reg_contains_mark,

                                          Label& done,

                                          Label* slow_case,

                                          BiasedLockingCounters* counters) {

   assert(UseBiasedLocking, "why call this otherwise?");

   assert(swap_reg == rax, "swap_reg must be rax, for cmpxchg");

   assert_different_registers(lock_reg, obj_reg, swap_reg);

   if (PrintBiasedLockingStatistics && counters == NULL)

     counters = BiasedLocking::counters();

   bool need_tmp_reg = false;

   if (tmp_reg == noreg) {

     need_tmp_reg = true;

     tmp_reg = lock_reg;

   } else {

     assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);

   }

   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");

   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());

   Address klass_addr     (obj_reg, oopDesc::klass_offset_in_bytes());

   Address saved_mark_addr(lock_reg, 0);

   // Biased locking

   // See whether the lock is currently biased toward our thread and

   // whether the epoch is still valid

   // Note that the runtime guarantees sufficient alignment of JavaThread

   // pointers to allow age to be placed into low bits

   // First check to see whether biasing is even enabled for this object

   Label cas_label;

   int null_check_offset = -1;

   if (!swap_reg_contains_mark) {

     null_check_offset = offset();

     movl(swap_reg, mark_addr);

   }

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   movl(tmp_reg, swap_reg);

   andl(tmp_reg, markOopDesc::biased_lock_mask_in_place);

   cmpl(tmp_reg, markOopDesc::biased_lock_pattern);

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   jcc(Assembler::notEqual, cas_label);

   // The bias pattern is present in the object's header. Need to check

   // whether the bias owner and the epoch are both still current.

   // Note that because there is no current thread register on x86 we

   // need to store off the mark word we read out of the object to

   // avoid reloading it and needing to recheck invariants below. This

   // store is unfortunate but it makes the overall code shorter and

   // simpler.

   movl(saved_mark_addr, swap_reg);

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   get_thread(tmp_reg);

   xorl(swap_reg, tmp_reg);

   if (swap_reg_contains_mark) {

     null_check_offset = offset();

   }

   movl(tmp_reg, klass_addr);

   xorl(swap_reg, Address(tmp_reg, Klass::prototype_header_offset()));

   andl(swap_reg, ~((int) markOopDesc::age_mask_in_place));

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address)counters->biased_lock_entry_count_addr()));

   }

   jcc(Assembler::equal, done);

   Label try_revoke_bias;

   Label try_rebias;

   // At this point we know that the header has the bias pattern and

   // that we are not the bias owner in the current epoch. We need to

   // figure out more details about the state of the header in order to

   // know what operations can be legally performed on the object's

   // header.

   // If the low three bits in the xor result aren't clear, that means

   // the prototype header is no longer biased and we have to revoke

   // the bias on this object.

   testl(swap_reg, markOopDesc::biased_lock_mask_in_place);

   jcc(Assembler::notZero, try_revoke_bias);

   // Biasing is still enabled for this data type. See whether the

   // epoch of the current bias is still valid, meaning that the epoch

   // bits of the mark word are equal to the epoch bits of the

   // prototype header. (Note that the prototype header's epoch bits

   // only change at a safepoint.) If not, attempt to rebias the object

   // toward the current thread. Note that we must be absolutely sure

   // that the current epoch is invalid in order to do this because

   // otherwise the manipulations it performs on the mark word are

   // illegal.

   testl(swap_reg, markOopDesc::epoch_mask_in_place);

   jcc(Assembler::notZero, try_rebias);

   // The epoch of the current bias is still valid but we know nothing

   // about the owner; it might be set or it might be clear. Try to

   // acquire the bias of the object using an atomic operation. If this

   // fails we will go in to the runtime to revoke the object's bias.

   // Note that we first construct the presumed unbiased header so we

   // don't accidentally blow away another thread's valid bias.

   movl(swap_reg, saved_mark_addr);

   andl(swap_reg,

        markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   get_thread(tmp_reg);

   orl(tmp_reg, swap_reg);

   if (os::is_MP()) {

     lock();

   }

   cmpxchgptr(tmp_reg, Address(obj_reg, 0));

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   // If the biasing toward our thread failed, this means that

   // another thread succeeded in biasing it toward itself and we

   // need to revoke that bias. The revocation will occur in the

   // interpreter runtime in the slow case.

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address)counters->anonymously_biased_lock_entry_count_addr()));

   }

   if (slow_case != NULL) {

     jcc(Assembler::notZero, *slow_case);

   }

   jmp(done);

   bind(try_rebias);

   // At this point we know the epoch has expired, meaning that the

   // current "bias owner", if any, is actually invalid. Under these

   // circumstances _only_, we are allowed to use the current header's

   // value as the comparison value when doing the cas to acquire the

   // bias in the current epoch. In other words, we allow transfer of

   // the bias from one thread to another directly in this situation.

   //

   // FIXME: due to a lack of registers we currently blow away the age

   // bits in this situation. Should attempt to preserve them.

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   get_thread(tmp_reg);

   movl(swap_reg, klass_addr);

   orl(tmp_reg, Address(swap_reg, Klass::prototype_header_offset()));

   movl(swap_reg, saved_mark_addr);

   if (os::is_MP()) {

     lock();

   }

   cmpxchgptr(tmp_reg, Address(obj_reg, 0));

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   // If the biasing toward our thread failed, then another thread

   // succeeded in biasing it toward itself and we need to revoke that

   // bias. The revocation will occur in the runtime in the slow case.

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address)counters->rebiased_lock_entry_count_addr()));

   }

   if (slow_case != NULL) {

     jcc(Assembler::notZero, *slow_case);

   }

   jmp(done);

   bind(try_revoke_bias);

   // The prototype mark in the klass doesn't have the bias bit set any

   // more, indicating that objects of this data type are not supposed

   // to be biased any more. We are going to try to reset the mark of

   // this object to the prototype value and fall through to the

   // CAS-based locking scheme. Note that if our CAS fails, it means

   // that another thread raced us for the privilege of revoking the

   // bias of this particular object, so it's okay to continue in the

   // normal locking code.

   //

   // FIXME: due to a lack of registers we currently blow away the age

   // bits in this situation. Should attempt to preserve them.

   movl(swap_reg, saved_mark_addr);

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   movl(tmp_reg, klass_addr);

   movl(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset()));

   if (os::is_MP()) {

     lock();

   }

   cmpxchgptr(tmp_reg, Address(obj_reg, 0));

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   // Fall through to the normal CAS-based lock, because no matter what

   // the result of the above CAS, some thread must have succeeded in

   // removing the bias bit from the object's header.

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address)counters->revoked_lock_entry_count_addr()));

   }

   bind(cas_label);

   return null_check_offset;

 }

 void MacroAssembler::call_VM_leaf_base(address entry_point,

                                        int number_of_arguments) {

   call(RuntimeAddress(entry_point));

   increment(rsp, number_of_arguments * wordSize);

 }

 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {

   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {

   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::cmpoop(Address src1, jobject obj) {

   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::cmpoop(Register src1, jobject obj) {

   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::extend_sign(Register hi, Register lo) {

   // According to Intel Doc. AP-526, "Integer Divide", p.18.

   if (VM_Version::is_P6() && hi == rdx && lo == rax) {

     cdql();

   } else {

     movl(hi, lo);

     sarl(hi, 31);

   }

 }

 void MacroAssembler::jC2(Register tmp, Label& L) {

   // set parity bit if FPU flag C2 is set (via rax)

   save_rax(tmp);

   fwait(); fnstsw_ax();

   sahf();

   restore_rax(tmp);

   // branch

   jcc(Assembler::parity, L);

 }

 void MacroAssembler::jnC2(Register tmp, Label& L) {

   // set parity bit if FPU flag C2 is set (via rax)

   save_rax(tmp);

   fwait(); fnstsw_ax();

   sahf();

   restore_rax(tmp);

   // branch

   jcc(Assembler::noParity, L);

 }

 // 32bit can do a case table jump in one instruction but we no longer allow the base

 // to be installed in the Address class

 void MacroAssembler::jump(ArrayAddress entry) {

   jmp(as_Address(entry));

 }

 // Note: y_lo will be destroyed

 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {

   // Long compare for Java (semantics as described in JVM spec.)

   Label high, low, done;

   cmpl(x_hi, y_hi);

   jcc(Assembler::less, low);

   jcc(Assembler::greater, high);

   // x_hi is the return register

   xorl(x_hi, x_hi);

   cmpl(x_lo, y_lo);

   jcc(Assembler::below, low);

   jcc(Assembler::equal, done);

   bind(high);

   xorl(x_hi, x_hi);

   increment(x_hi);

   jmp(done);

   bind(low);

   xorl(x_hi, x_hi);

   decrementl(x_hi);

   bind(done);

 }

 void MacroAssembler::lea(Register dst, AddressLiteral src) {

     mov_literal32(dst, (int32_t)src.target(), src.rspec());

 }

 void MacroAssembler::lea(Address dst, AddressLiteral adr) {

   // leal(dst, as_Address(adr));

   // see note in movl as to why we must use a move

   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());

 }

 void MacroAssembler::leave() {

   mov(rsp, rbp);

   pop(rbp);

 }

 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {

   // Multiplication of two Java long values stored on the stack

   // as illustrated below. Result is in rdx:rax.

   //

   // rsp ---> [  ??  ] \               \

   //            ....    | y_rsp_offset  |

   //          [ y_lo ] /  (in bytes)    | x_rsp_offset

   //          [ y_hi ]                  | (in bytes)

   //            ....                    |

   //          [ x_lo ]                 /

   //          [ x_hi ]

   //            ....

   //

   // Basic idea: lo(result) = lo(x_lo * y_lo)

   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)

   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);

   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);

   Label quick;

   // load x_hi, y_hi and check if quick

   // multiplication is possible

   movl(rbx, x_hi);

   movl(rcx, y_hi);

   movl(rax, rbx);

   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0

   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply

   // do full multiplication

   // 1st step

   mull(y_lo);                                    // x_hi * y_lo

   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,

   // 2nd step

   movl(rax, x_lo);

   mull(rcx);                                     // x_lo * y_hi

   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,

   // 3rd step

   bind(quick);                                   // note: rbx, = 0 if quick multiply!

   movl(rax, x_lo);

   mull(y_lo);                                    // x_lo * y_lo

   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)

 }

 void MacroAssembler::lneg(Register hi, Register lo) {

   negl(lo);

   adcl(hi, 0);

   negl(hi);

 }

 void MacroAssembler::lshl(Register hi, Register lo) {

   // Java shift left long support (semantics as described in JVM spec., p.305)

   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))

   // shift value is in rcx !

   assert(hi != rcx, "must not use rcx");

   assert(lo != rcx, "must not use rcx");

   const Register s = rcx;                        // shift count

   const int      n = BitsPerWord;

   Label L;

   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)

   cmpl(s, n);                                    // if (s < n)

   jcc(Assembler::less, L);                       // else (s >= n)

   movl(hi, lo);                                  // x := x << n

   xorl(lo, lo);

   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!

   bind(L);                                       // s (mod n) < n

   shldl(hi, lo);                                 // x := x << s

   shll(lo);

 }

 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {

   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)

   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))

   assert(hi != rcx, "must not use rcx");

   assert(lo != rcx, "must not use rcx");

   const Register s = rcx;                        // shift count

   const int      n = BitsPerWord;

   Label L;

   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)

   cmpl(s, n);                                    // if (s < n)

   jcc(Assembler::less, L);                       // else (s >= n)

   movl(lo, hi);                                  // x := x >> n

   if (sign_extension) sarl(hi, 31);

   else                xorl(hi, hi);

   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!

   bind(L);                                       // s (mod n) < n

   shrdl(lo, hi);                                 // x := x >> s

   if (sign_extension) sarl(hi);

   else                shrl(hi);

 }

 void MacroAssembler::movoop(Register dst, jobject obj) {

   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::movoop(Address dst, jobject obj) {

   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {

   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {

   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::movptr(Register dst, AddressLiteral src) {

   if (src.is_lval()) {

     mov_literal32(dst, (intptr_t)src.target(), src.rspec());

   } else {

     movl(dst, as_Address(src));

   }

 }

 void MacroAssembler::movptr(ArrayAddress dst, Register src) {

   movl(as_Address(dst), src);

 }

 void MacroAssembler::movptr(Register dst, ArrayAddress src) {

   movl(dst, as_Address(src));

 }

 // src should NEVER be a real pointer. Use AddressLiteral for true pointers

 void MacroAssembler::movptr(Address dst, intptr_t src) {

   movl(dst, src);

 }

 void MacroAssembler::pop_callee_saved_registers() {

   pop(rcx);

   pop(rdx);

   pop(rdi);

   pop(rsi);

 }

 void MacroAssembler::pop_fTOS() {

   fld_d(Address(rsp, 0));

   addl(rsp, 2 * wordSize);

 }

 void MacroAssembler::push_callee_saved_registers() {

   push(rsi);

   push(rdi);

   push(rdx);

   push(rcx);

 }

 void MacroAssembler::push_fTOS() {

   subl(rsp, 2 * wordSize);

   fstp_d(Address(rsp, 0));

 }

 void MacroAssembler::pushoop(jobject obj) {

   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::pushklass(Metadata* obj) {

   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::pushptr(AddressLiteral src) {

   if (src.is_lval()) {

     push_literal32((int32_t)src.target(), src.rspec());

   } else {

     pushl(as_Address(src));

   }

 }

 void MacroAssembler::set_word_if_not_zero(Register dst) {

   xorl(dst, dst);

   set_byte_if_not_zero(dst);

 }

 static void pass_arg0(MacroAssembler* masm, Register arg) {

   masm->push(arg);

 }

 static void pass_arg1(MacroAssembler* masm, Register arg) {

   masm->push(arg);

 }

 static void pass_arg2(MacroAssembler* masm, Register arg) {

   masm->push(arg);

 }

 static void pass_arg3(MacroAssembler* masm, Register arg) {

   masm->push(arg);

 }

 #ifndef PRODUCT

 extern "C" void findpc(intptr_t x);

 #endif

 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {

   // In order to get locks to work, we need to fake a in_VM state

   JavaThread* thread = JavaThread::current();

   JavaThreadState saved_state = thread->thread_state();

   thread->set_thread_state(_thread_in_vm);

   if (ShowMessageBoxOnError) {

     JavaThread* thread = JavaThread::current();

     JavaThreadState saved_state = thread->thread_state();

     thread->set_thread_state(_thread_in_vm);

     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {

       ttyLocker ttyl;

       BytecodeCounter::print();

     }

     // To see where a verify_oop failed, get $ebx+40/X for this frame.

     // This is the value of eip which points to where verify_oop will return.

     if (os::message_box(msg, "Execution stopped, print registers?")) {

       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);

       BREAKPOINT;

     }

   } else {

     ttyLocker ttyl;

     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);

   }

   // Don't assert holding the ttyLock

     assert(false, err_msg("DEBUG MESSAGE: %s", msg));

   ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);

 }

 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {

   ttyLocker ttyl;

   FlagSetting fs(Debugging, true);

   tty->print_cr("eip = 0x%08x", eip);

 #ifndef PRODUCT

   if ((WizardMode || Verbose) && PrintMiscellaneous) {

     tty->cr();

     findpc(eip);

     tty->cr();

   }

 #endif

 #define PRINT_REG(rax) \

   { tty->print("%s = ", #rax); os::print_location(tty, rax); }

   PRINT_REG(rax);

   PRINT_REG(rbx);

   PRINT_REG(rcx);

   PRINT_REG(rdx);

   PRINT_REG(rdi);

   PRINT_REG(rsi);

   PRINT_REG(rbp);

   PRINT_REG(rsp);

 #undef PRINT_REG

   // Print some words near top of staack.

   int* dump_sp = (int*) rsp;

   for (int col1 = 0; col1 < 8; col1++) {

     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);

     os::print_location(tty, *dump_sp++);

   }

   for (int row = 0; row < 16; row++) {

     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);

     for (int col = 0; col < 8; col++) {

       tty->print(" 0x%08x", *dump_sp++);

     }

     tty->cr();

   }

   // Print some instructions around pc:

   Disassembler::decode((address)eip-64, (address)eip);

   tty->print_cr("--------");

   Disassembler::decode((address)eip, (address)eip+32);

 }

 void MacroAssembler::stop(const char* msg) {

   ExternalAddress message((address)msg);

   // push address of message

   pushptr(message.addr());

   { Label L; call(L, relocInfo::none); bind(L); }     // push eip

   pusha();                                            // push registers

   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));

   hlt();

 }

 void MacroAssembler::warn(const char* msg) {

   push_CPU_state();

   ExternalAddress message((address) msg);

   // push address of message

   pushptr(message.addr());

   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));

   addl(rsp, wordSize);       // discard argument

   pop_CPU_state();

 }

 void MacroAssembler::print_state() {

   { Label L; call(L, relocInfo::none); bind(L); }     // push eip

   pusha();                                            // push registers

   push_CPU_state();

   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));

   pop_CPU_state();

   popa();

   addl(rsp, wordSize);

 }

 #else // _LP64

 // 64 bit versions

 Address MacroAssembler::as_Address(AddressLiteral adr) {

   // amd64 always does this as a pc-rel

   // we can be absolute or disp based on the instruction type

   // jmp/call are displacements others are absolute

   assert(!adr.is_lval(), "must be rval");

   assert(reachable(adr), "must be");

   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());

 }

 Address MacroAssembler::as_Address(ArrayAddress adr) {

   AddressLiteral base = adr.base();

   lea(rscratch1, base);

   Address index = adr.index();

   assert(index._disp == 0, "must not have disp"); // maybe it can?

   Address array(rscratch1, index._index, index._scale, index._disp);

   return array;

 }

 int MacroAssembler::biased_locking_enter(Register lock_reg,

                                          Register obj_reg,

                                          Register swap_reg,

                                          Register tmp_reg,

                                          bool swap_reg_contains_mark,

                                          Label& done,

                                          Label* slow_case,

                                          BiasedLockingCounters* counters) {

   assert(UseBiasedLocking, "why call this otherwise?");

   assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");

   assert(tmp_reg != noreg, "tmp_reg must be supplied");

   assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);

   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");

   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());

   Address saved_mark_addr(lock_reg, 0);

   if (PrintBiasedLockingStatistics && counters == NULL)

     counters = BiasedLocking::counters();

   // Biased locking

   // See whether the lock is currently biased toward our thread and

   // whether the epoch is still valid

   // Note that the runtime guarantees sufficient alignment of JavaThread

   // pointers to allow age to be placed into low bits

   // First check to see whether biasing is even enabled for this object

   Label cas_label;

   int null_check_offset = -1;

   if (!swap_reg_contains_mark) {

     null_check_offset = offset();

     movq(swap_reg, mark_addr);

   }

   movq(tmp_reg, swap_reg);

   andq(tmp_reg, markOopDesc::biased_lock_mask_in_place);

   cmpq(tmp_reg, markOopDesc::biased_lock_pattern);

   jcc(Assembler::notEqual, cas_label);

   // The bias pattern is present in the object's header. Need to check

   // whether the bias owner and the epoch are both still current.

   load_prototype_header(tmp_reg, obj_reg);

   orq(tmp_reg, r15_thread);

   xorq(tmp_reg, swap_reg);

   andq(tmp_reg, ~((int) markOopDesc::age_mask_in_place));

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));

   }

   jcc(Assembler::equal, done);

   Label try_revoke_bias;

   Label try_rebias;

   // At this point we know that the header has the bias pattern and

   // that we are not the bias owner in the current epoch. We need to

   // figure out more details about the state of the header in order to

   // know what operations can be legally performed on the object's

   // header.

   // If the low three bits in the xor result aren't clear, that means

   // the prototype header is no longer biased and we have to revoke

   // the bias on this object.

   testq(tmp_reg, markOopDesc::biased_lock_mask_in_place);

   jcc(Assembler::notZero, try_revoke_bias);

   // Biasing is still enabled for this data type. See whether the

   // epoch of the current bias is still valid, meaning that the epoch

   // bits of the mark word are equal to the epoch bits of the

   // prototype header. (Note that the prototype header's epoch bits

   // only change at a safepoint.) If not, attempt to rebias the object

   // toward the current thread. Note that we must be absolutely sure

   // that the current epoch is invalid in order to do this because

   // otherwise the manipulations it performs on the mark word are

   // illegal.

   testq(tmp_reg, markOopDesc::epoch_mask_in_place);

   jcc(Assembler::notZero, try_rebias);

   // The epoch of the current bias is still valid but we know nothing

   // about the owner; it might be set or it might be clear. Try to

   // acquire the bias of the object using an atomic operation. If this

   // fails we will go in to the runtime to revoke the object's bias.

   // Note that we first construct the presumed unbiased header so we

   // don't accidentally blow away another thread's valid bias.

   andq(swap_reg,

        markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);

   movq(tmp_reg, swap_reg);

   orq(tmp_reg, r15_thread);

   if (os::is_MP()) {

     lock();

   }

   cmpxchgq(tmp_reg, Address(obj_reg, 0));

   // If the biasing toward our thread failed, this means that

   // another thread succeeded in biasing it toward itself and we

   // need to revoke that bias. The revocation will occur in the

   // interpreter runtime in the slow case.

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));

   }

   if (slow_case != NULL) {

     jcc(Assembler::notZero, *slow_case);

   }

   jmp(done);

   bind(try_rebias);

   // At this point we know the epoch has expired, meaning that the

   // current "bias owner", if any, is actually invalid. Under these

   // circumstances _only_, we are allowed to use the current header's

   // value as the comparison value when doing the cas to acquire the

   // bias in the current epoch. In other words, we allow transfer of

   // the bias from one thread to another directly in this situation.

   //

   // FIXME: due to a lack of registers we currently blow away the age

   // bits in this situation. Should attempt to preserve them.

   load_prototype_header(tmp_reg, obj_reg);

   orq(tmp_reg, r15_thread);

   if (os::is_MP()) {

     lock();

   }

   cmpxchgq(tmp_reg, Address(obj_reg, 0));

   // If the biasing toward our thread failed, then another thread

   // succeeded in biasing it toward itself and we need to revoke that

   // bias. The revocation will occur in the runtime in the slow case.

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));

   }

   if (slow_case != NULL) {

     jcc(Assembler::notZero, *slow_case);

   }

   jmp(done);

   bind(try_revoke_bias);

   // The prototype mark in the klass doesn't have the bias bit set any

   // more, indicating that objects of this data type are not supposed

   // to be biased any more. We are going to try to reset the mark of

   // this object to the prototype value and fall through to the

   // CAS-based locking scheme. Note that if our CAS fails, it means

   // that another thread raced us for the privilege of revoking the

   // bias of this particular object, so it's okay to continue in the

   // normal locking code.

   //

   // FIXME: due to a lack of registers we currently blow away the age

   // bits in this situation. Should attempt to preserve them.

   load_prototype_header(tmp_reg, obj_reg);

   if (os::is_MP()) {

     lock();

   }

   cmpxchgq(tmp_reg, Address(obj_reg, 0));

   // Fall through to the normal CAS-based lock, because no matter what

   // the result of the above CAS, some thread must have succeeded in

   // removing the bias bit from the object's header.

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address) counters->revoked_lock_entry_count_addr()));

   }

   bind(cas_label);

   return null_check_offset;

 }

 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {

   Label L, E;

 #ifdef _WIN64

   // Windows always allocates space for it's register args

   assert(num_args <= 4, "only register arguments supported");

   subq(rsp,  frame::arg_reg_save_area_bytes);

 #endif

   // Align stack if necessary

   testl(rsp, 15);

   jcc(Assembler::zero, L);

   subq(rsp, 8);

   {

     call(RuntimeAddress(entry_point));

   }

   addq(rsp, 8);

   jmp(E);

   bind(L);

   {

     call(RuntimeAddress(entry_point));

   }

   bind(E);

 #ifdef _WIN64

   // restore stack pointer

   addq(rsp, frame::arg_reg_save_area_bytes);

 #endif

 }

 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {

   assert(!src2.is_lval(), "should use cmpptr");

   if (reachable(src2)) {

     cmpq(src1, as_Address(src2));

   } else {

     lea(rscratch1, src2);

     Assembler::cmpq(src1, Address(rscratch1, 0));

   }

 }

 int MacroAssembler::corrected_idivq(Register reg) {

   // Full implementation of Java ldiv and lrem; checks for special

   // case as described in JVM spec., p.243 & p.271.  The function

   // returns the (pc) offset of the idivl instruction - may be needed

   // for implicit exceptions.

   //

   //         normal case                           special case

   //

   // input : rax: dividend                         min_long

   //         reg: divisor   (may not be eax/edx)   -1

   //

   // output: rax: quotient  (= rax idiv reg)       min_long

   //         rdx: remainder (= rax irem reg)       0

   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");

   static const int64_t min_long = 0x8000000000000000;

   Label normal_case, special_case;

   // check for special case

   cmp64(rax, ExternalAddress((address) &min_long));

   jcc(Assembler::notEqual, normal_case);

   xorl(rdx, rdx); // prepare rdx for possible special case (where

                   // remainder = 0)

   cmpq(reg, -1);

   jcc(Assembler::equal, special_case);

   // handle normal case

   bind(normal_case);

   cdqq();

   int idivq_offset = offset();

   idivq(reg);

   // normal and special case exit

   bind(special_case);

   return idivq_offset;

 }

 void MacroAssembler::decrementq(Register reg, int value) {

   if (value == min_jint) { subq(reg, value); return; }

   if (value <  0) { incrementq(reg, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { decq(reg) ; return; }

   /* else */      { subq(reg, value)       ; return; }

 }

 void MacroAssembler::decrementq(Address dst, int value) {

   if (value == min_jint) { subq(dst, value); return; }

   if (value <  0) { incrementq(dst, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { decq(dst) ; return; }

   /* else */      { subq(dst, value)       ; return; }

 }

 void MacroAssembler::incrementq(Register reg, int value) {

   if (value == min_jint) { addq(reg, value); return; }

   if (value <  0) { decrementq(reg, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { incq(reg) ; return; }

   /* else */      { addq(reg, value)       ; return; }

 }

 void MacroAssembler::incrementq(Address dst, int value) {

   if (value == min_jint) { addq(dst, value); return; }

   if (value <  0) { decrementq(dst, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { incq(dst) ; return; }

   /* else */      { addq(dst, value)       ; return; }

 }

 // 32bit can do a case table jump in one instruction but we no longer allow the base

 // to be installed in the Address class

 void MacroAssembler::jump(ArrayAddress entry) {

   lea(rscratch1, entry.base());

   Address dispatch = entry.index();

   assert(dispatch._base == noreg, "must be");

   dispatch._base = rscratch1;

   jmp(dispatch);

 }

 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {

   ShouldNotReachHere(); // 64bit doesn't use two regs

   cmpq(x_lo, y_lo);

 }

 void MacroAssembler::lea(Register dst, AddressLiteral src) {

     mov_literal64(dst, (intptr_t)src.target(), src.rspec());

 }

 void MacroAssembler::lea(Address dst, AddressLiteral adr) {

   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());

   movptr(dst, rscratch1);

 }

 void MacroAssembler::leave() {

   // %%% is this really better? Why not on 32bit too?

   emit_byte(0xC9); // LEAVE

 }

 void MacroAssembler::lneg(Register hi, Register lo) {

   ShouldNotReachHere(); // 64bit doesn't use two regs

   negq(lo);

 }

 void MacroAssembler::movoop(Register dst, jobject obj) {

   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::movoop(Address dst, jobject obj) {

   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());

   movq(dst, rscratch1);

 }

 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {

   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {

   mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());

   movq(dst, rscratch1);

 }

 void MacroAssembler::movptr(Register dst, AddressLiteral src) {

   if (src.is_lval()) {

     mov_literal64(dst, (intptr_t)src.target(), src.rspec());

   } else {

     if (reachable(src)) {

       movq(dst, as_Address(src));

     } else {

       lea(rscratch1, src);

       movq(dst, Address(rscratch1,0));

     }

   }

 }

 void MacroAssembler::movptr(ArrayAddress dst, Register src) {

   movq(as_Address(dst), src);

 }

 void MacroAssembler::movptr(Register dst, ArrayAddress src) {

   movq(dst, as_Address(src));

 }

 // src should NEVER be a real pointer. Use AddressLiteral for true pointers

 void MacroAssembler::movptr(Address dst, intptr_t src) {

   mov64(rscratch1, src);

   movq(dst, rscratch1);

 }

 // These are mostly for initializing NULL

 void MacroAssembler::movptr(Address dst, int32_t src) {

   movslq(dst, src);

 }

 void MacroAssembler::movptr(Register dst, int32_t src) {

   mov64(dst, (intptr_t)src);

 }

 void MacroAssembler::pushoop(jobject obj) {

   movoop(rscratch1, obj);

   push(rscratch1);

 }

 void MacroAssembler::pushklass(Metadata* obj) {

   mov_metadata(rscratch1, obj);

   push(rscratch1);

 }

 void MacroAssembler::pushptr(AddressLiteral src) {

   lea(rscratch1, src);

   if (src.is_lval()) {

     push(rscratch1);

   } else {

     pushq(Address(rscratch1, 0));

   }

 }

 void MacroAssembler::reset_last_Java_frame(bool clear_fp,

                                            bool clear_pc) {

   // we must set sp to zero to clear frame

   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);

   // must clear fp, so that compiled frames are not confused; it is

   // possible that we need it only for debugging

   if (clear_fp) {

     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);

   }

   if (clear_pc) {

     movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);

   }

 }

 void MacroAssembler::set_last_Java_frame(Register last_java_sp,

                                          Register last_java_fp,

                                          address  last_java_pc) {

   // determine last_java_sp register

   if (!last_java_sp->is_valid()) {

     last_java_sp = rsp;

   }

   // last_java_fp is optional

   if (last_java_fp->is_valid()) {

     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),

            last_java_fp);

   }

   // last_java_pc is optional

   if (last_java_pc != NULL) {

     Address java_pc(r15_thread,

                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());

     lea(rscratch1, InternalAddress(last_java_pc));

     movptr(java_pc, rscratch1);

   }

   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);

 }

 static void pass_arg0(MacroAssembler* masm, Register arg) {

   if (c_rarg0 != arg ) {

     masm->mov(c_rarg0, arg);

   }

 }

 static void pass_arg1(MacroAssembler* masm, Register arg) {

   if (c_rarg1 != arg ) {

     masm->mov(c_rarg1, arg);

   }

 }

 static void pass_arg2(MacroAssembler* masm, Register arg) {

   if (c_rarg2 != arg ) {

     masm->mov(c_rarg2, arg);

   }

 }

 static void pass_arg3(MacroAssembler* masm, Register arg) {

   if (c_rarg3 != arg ) {

     masm->mov(c_rarg3, arg);

   }

 }

 void MacroAssembler::stop(const char* msg) {

   address rip = pc();

   pusha(); // get regs on stack

   lea(c_rarg0, ExternalAddress((address) msg));

   lea(c_rarg1, InternalAddress(rip));

   movq(c_rarg2, rsp); // pass pointer to regs array

   andq(rsp, -16); // align stack as required by ABI

   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));

   hlt();

 }

 void MacroAssembler::warn(const char* msg) {

   push(rbp);

   movq(rbp, rsp);

   andq(rsp, -16);     // align stack as required by push_CPU_state and call

   push_CPU_state();   // keeps alignment at 16 bytes

   lea(c_rarg0, ExternalAddress((address) msg));

   call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0);

   pop_CPU_state();

   mov(rsp, rbp);

   pop(rbp);

 }

 void MacroAssembler::print_state() {

   address rip = pc();

   pusha();            // get regs on stack

   push(rbp);

   movq(rbp, rsp);

   andq(rsp, -16);     // align stack as required by push_CPU_state and call

   push_CPU_state();   // keeps alignment at 16 bytes

   lea(c_rarg0, InternalAddress(rip));

   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array

   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);

   pop_CPU_state();

   mov(rsp, rbp);

   pop(rbp);

   popa();

 }

 #ifndef PRODUCT

 extern "C" void findpc(intptr_t x);

 #endif

 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {

   // In order to get locks to work, we need to fake a in_VM state

   if (ShowMessageBoxOnError) {

     JavaThread* thread = JavaThread::current();

     JavaThreadState saved_state = thread->thread_state();

     thread->set_thread_state(_thread_in_vm);

 #ifndef PRODUCT

     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {

       ttyLocker ttyl;

       BytecodeCounter::print();

     }

 #endif

     // To see where a verify_oop failed, get $ebx+40/X for this frame.

     // XXX correct this offset for amd64

     // This is the value of eip which points to where verify_oop will return.

     if (os::message_box(msg, "Execution stopped, print registers?")) {

       print_state64(pc, regs);

       BREAKPOINT;

       assert(false, "start up GDB");

     }

     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);

   } else {

     ttyLocker ttyl;

     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",

                     msg);

     assert(false, err_msg("DEBUG MESSAGE: %s", msg));

   }

 }

 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {

   ttyLocker ttyl;

   FlagSetting fs(Debugging, true);

   tty->print_cr("rip = 0x%016lx", pc);

 #ifndef PRODUCT

   tty->cr();

   findpc(pc);

   tty->cr();

 #endif

 #define PRINT_REG(rax, value) \

   { tty->print("%s = ", #rax); os::print_location(tty, value); }

   PRINT_REG(rax, regs[15]);

   PRINT_REG(rbx, regs[12]);

   PRINT_REG(rcx, regs[14]);

   PRINT_REG(rdx, regs[13]);

   PRINT_REG(rdi, regs[8]);

   PRINT_REG(rsi, regs[9]);

   PRINT_REG(rbp, regs[10]);

   PRINT_REG(rsp, regs[11]);

   PRINT_REG(r8 , regs[7]);

   PRINT_REG(r9 , regs[6]);

   PRINT_REG(r10, regs[5]);

   PRINT_REG(r11, regs[4]);

   PRINT_REG(r12, regs[3]);

   PRINT_REG(r13, regs[2]);

   PRINT_REG(r14, regs[1]);

   PRINT_REG(r15, regs[0]);

 #undef PRINT_REG

   // Print some words near top of staack.

   int64_t* rsp = (int64_t*) regs[11];

   int64_t* dump_sp = rsp;

   for (int col1 = 0; col1 < 8; col1++) {

     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp);

     os::print_location(tty, *dump_sp++);

   }

   for (int row = 0; row < 25; row++) {

     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp);

     for (int col = 0; col < 4; col++) {

       tty->print(" 0x%016lx", *dump_sp++);

     }

     tty->cr();

   }

   // Print some instructions around pc:

   Disassembler::decode((address)pc-64, (address)pc);

   tty->print_cr("--------");

   Disassembler::decode((address)pc, (address)pc+32);

 }

 #endif // _LP64

 // Now versions that are common to 32/64 bit

 void MacroAssembler::addptr(Register dst, int32_t imm32) {

   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));

 }

 void MacroAssembler::addptr(Register dst, Register src) {

   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));

 }

 void MacroAssembler::addptr(Address dst, Register src) {

   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));

 }

 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {

   if (reachable(src)) {

     Assembler::addsd(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::addsd(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {

   if (reachable(src)) {

     addss(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     addss(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::align(int modulus) {

   if (offset() % modulus != 0) {

     nop(modulus - (offset() % modulus));

   }

 }

 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {

   // Used in sign-masking with aligned address.

   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");

   if (reachable(src)) {

     Assembler::andpd(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::andpd(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {

   // Used in sign-masking with aligned address.

   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");

   if (reachable(src)) {

     Assembler::andps(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::andps(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::andptr(Register dst, int32_t imm32) {

   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));

 }

 void MacroAssembler::atomic_incl(AddressLiteral counter_addr) {

   pushf();

   if (os::is_MP())

     lock();

   incrementl(counter_addr);

   popf();

 }

 // Writes to stack successive pages until offset reached to check for

 // stack overflow + shadow pages.  This clobbers tmp.

 void MacroAssembler::bang_stack_size(Register size, Register tmp) {

   movptr(tmp, rsp);

   // Bang stack for total size given plus shadow page size.

   // Bang one page at a time because large size can bang beyond yellow and

   // red zones.

   Label loop;

   bind(loop);

   movl(Address(tmp, (-os::vm_page_size())), size );

   subptr(tmp, os::vm_page_size());

   subl(size, os::vm_page_size());

   jcc(Assembler::greater, loop);

   // Bang down shadow pages too.

   // The -1 because we already subtracted 1 page.

   for (int i = 0; i< StackShadowPages-1; i++) {

     // this could be any sized move but this is can be a debugging crumb

     // so the bigger the better.

     movptr(Address(tmp, (-i*os::vm_page_size())), size );

   }

 }

 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {

   assert(UseBiasedLocking, "why call this otherwise?");

   // Check for biased locking unlock case, which is a no-op

   // Note: we do not have to check the thread ID for two reasons.

   // First, the interpreter checks for IllegalMonitorStateException at

   // a higher level. Second, if the bias was revoked while we held the

   // lock, the object could not be rebiased toward another thread, so

   // the bias bit would be clear.

   movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));

   andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);

   cmpptr(temp_reg, markOopDesc::biased_lock_pattern);

   jcc(Assembler::equal, done);

 }

 void MacroAssembler::c2bool(Register x) {

   // implements x == 0 ? 0 : 1

   // note: must only look at least-significant byte of x

   //       since C-style booleans are stored in one byte

   //       only! (was bug)

   andl(x, 0xFF);

   setb(Assembler::notZero, x);

 }

 // Wouldn't need if AddressLiteral version had new name

 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {

   Assembler::call(L, rtype);

 }

 void MacroAssembler::call(Register entry) {

   Assembler::call(entry);

 }

 void MacroAssembler::call(AddressLiteral entry) {

   if (reachable(entry)) {

     Assembler::call_literal(entry.target(), entry.rspec());

   } else {

     lea(rscratch1, entry);

     Assembler::call(rscratch1);

   }

 }

 void MacroAssembler::ic_call(address entry) {

   RelocationHolder rh = virtual_call_Relocation::spec(pc());

   movptr(rax, (intptr_t)Universe::non_oop_word());

   call(AddressLiteral(entry, rh));

 }

 // Implementation of call_VM versions

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              bool check_exceptions) {

   Label C, E;

   call(C, relocInfo::none);

   jmp(E);

   bind(C);

   call_VM_helper(oop_result, entry_point, 0, check_exceptions);

   ret(0);

   bind(E);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              Register arg_1,

                              bool check_exceptions) {

   Label C, E;

   call(C, relocInfo::none);

   jmp(E);

   bind(C);

   pass_arg1(this, arg_1);

   call_VM_helper(oop_result, entry_point, 1, check_exceptions);

   ret(0);

   bind(E);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              bool check_exceptions) {

   Label C, E;

   call(C, relocInfo::none);

   jmp(E);

   bind(C);

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   call_VM_helper(oop_result, entry_point, 2, check_exceptions);

   ret(0);

   bind(E);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              Register arg_3,

                              bool check_exceptions) {

   Label C, E;

   call(C, relocInfo::none);

   jmp(E);

   bind(C);

   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));

   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));

   pass_arg3(this, arg_3);

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   call_VM_helper(oop_result, entry_point, 3, check_exceptions);

   ret(0);

   bind(E);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              int number_of_arguments,

                              bool check_exceptions) {

   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);

   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              Register arg_1,

                              bool check_exceptions) {

   pass_arg1(this, arg_1);

   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              bool check_exceptions) {

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              Register arg_3,

                              bool check_exceptions) {

   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));

   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));

   pass_arg3(this, arg_3);

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);

 }

 void MacroAssembler::super_call_VM(Register oop_result,

                                    Register last_java_sp,

                                    address entry_point,

                                    int number_of_arguments,

                                    bool check_exceptions) {

   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);

   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);

 }

 void MacroAssembler::super_call_VM(Register oop_result,

                                    Register last_java_sp,

                                    address entry_point,

                                    Register arg_1,

                                    bool check_exceptions) {

   pass_arg1(this, arg_1);

   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);

 }

 void MacroAssembler::super_call_VM(Register oop_result,

                                    Register last_java_sp,

                                    address entry_point,

                                    Register arg_1,

                                    Register arg_2,

                                    bool check_exceptions) {

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);

 }

 void MacroAssembler::super_call_VM(Register oop_result,

                                    Register last_java_sp,

                                    address entry_point,

                                    Register arg_1,

                                    Register arg_2,

                                    Register arg_3,

                                    bool check_exceptions) {

   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));

   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));

   pass_arg3(this, arg_3);

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);

 }

 void MacroAssembler::call_VM_base(Register oop_result,

                                   Register java_thread,

                                   Register last_java_sp,

                                   address  entry_point,

                                   int      number_of_arguments,

                                   bool     check_exceptions) {

   // determine java_thread register

   if (!java_thread->is_valid()) {

 #ifdef _LP64

     java_thread = r15_thread;

 #else

     java_thread = rdi;

     get_thread(java_thread);

 #endif // LP64

   }

   // determine last_java_sp register

   if (!last_java_sp->is_valid()) {

     last_java_sp = rsp;

   }

   // debugging support

   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");

   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));

 #ifdef ASSERT

   // TraceBytecodes does not use r12 but saves it over the call, so don't verify

   // r12 is the heapbase.

   LP64_ONLY(if ((UseCompressedOops || UseCompressedKlassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)

 #endif // ASSERT

   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");

   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");

   // push java thread (becomes first argument of C function)

   NOT_LP64(push(java_thread); number_of_arguments++);

   LP64_ONLY(mov(c_rarg0, r15_thread));

   // set last Java frame before call

   assert(last_java_sp != rbp, "can't use ebp/rbp");

   // Only interpreter should have to set fp

   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);

   // do the call, remove parameters

   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);

   // restore the thread (cannot use the pushed argument since arguments

   // may be overwritten by C code generated by an optimizing compiler);

   // however can use the register value directly if it is callee saved.

   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {

     // rdi & rsi (also r15) are callee saved -> nothing to do

 #ifdef ASSERT

     guarantee(java_thread != rax, "change this code");

     push(rax);

     { Label L;

       get_thread(rax);

       cmpptr(java_thread, rax);

       jcc(Assembler::equal, L);

       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");

       bind(L);

     }

     pop(rax);

 #endif

   } else {

     get_thread(java_thread);

   }

   // reset last Java frame

   // Only interpreter should have to clear fp

   reset_last_Java_frame(java_thread, true, false);

 #ifndef CC_INTERP

    // C++ interp handles this in the interpreter

   check_and_handle_popframe(java_thread);

   check_and_handle_earlyret(java_thread);

 #endif /* CC_INTERP */

   if (check_exceptions) {

     // check for pending exceptions (java_thread is set upon return)

     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);

 #ifndef _LP64

     jump_cc(Assembler::notEqual,

             RuntimeAddress(StubRoutines::forward_exception_entry()));

 #else

     // This used to conditionally jump to forward_exception however it is

     // possible if we relocate that the branch will not reach. So we must jump

     // around so we can always reach

     Label ok;

     jcc(Assembler::equal, ok);

     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));

     bind(ok);

 #endif // LP64

   }

   // get oop result if there is one and reset the value in the thread

   if (oop_result->is_valid()) {

     get_vm_result(oop_result, java_thread);

   }

 }

 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {

   // Calculate the value for last_Java_sp

   // somewhat subtle. call_VM does an intermediate call

   // which places a return address on the stack just under the

   // stack pointer as the user finsihed with it. This allows

   // use to retrieve last_Java_pc from last_Java_sp[-1].

   // On 32bit we then have to push additional args on the stack to accomplish

   // the actual requested call. On 64bit call_VM only can use register args

   // so the only extra space is the return address that call_VM created.

   // This hopefully explains the calculations here.

 #ifdef _LP64

   // We've pushed one address, correct last_Java_sp

   lea(rax, Address(rsp, wordSize));

 #else

   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));

 #endif // LP64

   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {

   call_VM_leaf_base(entry_point, number_of_arguments);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {

   pass_arg0(this, arg_0);

   call_VM_leaf(entry_point, 1);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {

   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));

   pass_arg1(this, arg_1);

   pass_arg0(this, arg_0);

   call_VM_leaf(entry_point, 2);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {

   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));

   pass_arg1(this, arg_1);

   pass_arg0(this, arg_0);

   call_VM_leaf(entry_point, 3);

 }

 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {

   pass_arg0(this, arg_0);

   MacroAssembler::call_VM_leaf_base(entry_point, 1);

 }

 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {

   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));

   pass_arg1(this, arg_1);

   pass_arg0(this, arg_0);

   MacroAssembler::call_VM_leaf_base(entry_point, 2);

 }

 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {

   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));

   pass_arg1(this, arg_1);

   pass_arg0(this, arg_0);

   MacroAssembler::call_VM_leaf_base(entry_point, 3);

 }

 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {

   LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg"));

   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));

   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));

   pass_arg3(this, arg_3);

   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));

   pass_arg1(this, arg_1);

   pass_arg0(this, arg_0);

   MacroAssembler::call_VM_leaf_base(entry_point, 4);

 }

 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {

   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));

   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);

   verify_oop(oop_result, "broken oop in call_VM_base");

 }

 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {

   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));

   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);

 }

 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {

 }

 void MacroAssembler::check_and_handle_popframe(Register java_thread) {

 }

 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {

   if (reachable(src1)) {

     cmpl(as_Address(src1), imm);

   } else {

     lea(rscratch1, src1);

     cmpl(Address(rscratch1, 0), imm);

   }

 }

 void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {

   assert(!src2.is_lval(), "use cmpptr");

   if (reachable(src2)) {

     cmpl(src1, as_Address(src2));

   } else {

     lea(rscratch1, src2);

     cmpl(src1, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::cmp32(Register src1, int32_t imm) {

   Assembler::cmpl(src1, imm);

 }

 void MacroAssembler::cmp32(Register src1, Address src2) {

   Assembler::cmpl(src1, src2);

 }

 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {

   ucomisd(opr1, opr2);

   Label L;

   if (unordered_is_less) {

     movl(dst, -1);

     jcc(Assembler::parity, L);

     jcc(Assembler::below , L);

     movl(dst, 0);

     jcc(Assembler::equal , L);

     increment(dst);

   } else { // unordered is greater

     movl(dst, 1);

     jcc(Assembler::parity, L);

     jcc(Assembler::above , L);

     movl(dst, 0);

     jcc(Assembler::equal , L);

     decrementl(dst);

   }

   bind(L);

 }

 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {

   ucomiss(opr1, opr2);

   Label L;

   if (unordered_is_less) {

     movl(dst, -1);

     jcc(Assembler::parity, L);

     jcc(Assembler::below , L);

     movl(dst, 0);

     jcc(Assembler::equal , L);

     increment(dst);

   } else { // unordered is greater

     movl(dst, 1);

     jcc(Assembler::parity, L);

     jcc(Assembler::above , L);

     movl(dst, 0);

     jcc(Assembler::equal , L);

     decrementl(dst);

   }

   bind(L);

 }

 void MacroAssembler::cmp8(AddressLiteral src1, int imm) {

   if (reachable(src1)) {

     cmpb(as_Address(src1), imm);

   } else {

     lea(rscratch1, src1);

     cmpb(Address(rscratch1, 0), imm);

   }

 }

 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {

 #ifdef _LP64

   if (src2.is_lval()) {

     movptr(rscratch1, src2);

     Assembler::cmpq(src1, rscratch1);

   } else if (reachable(src2)) {

     cmpq(src1, as_Address(src2));

   } else {

     lea(rscratch1, src2);

     Assembler::cmpq(src1, Address(rscratch1, 0));

   }

 #else

   if (src2.is_lval()) {

     cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());

   } else {

     cmpl(src1, as_Address(src2));

   }

 #endif // _LP64

 }

 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {

   assert(src2.is_lval(), "not a mem-mem compare");

 #ifdef _LP64

   // moves src2's literal address

   movptr(rscratch1, src2);

   Assembler::cmpq(src1, rscratch1);

 #else

   cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());

 #endif // _LP64

 }

 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {

   if (reachable(adr)) {

     if (os::is_MP())

       lock();

     cmpxchgptr(reg, as_Address(adr));

   } else {

     lea(rscratch1, adr);

     if (os::is_MP())

       lock();

     cmpxchgptr(reg, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {

   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));

 }

 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {

   if (reachable(src)) {

     Assembler::comisd(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::comisd(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {

   if (reachable(src)) {

     Assembler::comiss(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::comiss(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {

   Condition negated_cond = negate_condition(cond);

   Label L;

   jcc(negated_cond, L);

   atomic_incl(counter_addr);

   bind(L);

 }

 int MacroAssembler::corrected_idivl(Register reg) {

   // Full implementation of Java idiv and irem; checks for

   // special case as described in JVM spec., p.243 & p.271.

   // The function returns the (pc) offset of the idivl

   // instruction - may be needed for implicit exceptions.

   //

   //         normal case                           special case

   //

   // input : rax,: dividend                         min_int

   //         reg: divisor   (may not be rax,/rdx)   -1

   //

   // output: rax,: quotient  (= rax, idiv reg)       min_int

   //         rdx: remainder (= rax, irem reg)       0

   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");

   const int min_int = 0x80000000;

   Label normal_case, special_case;

   // check for special case

   cmpl(rax, min_int);

   jcc(Assembler::notEqual, normal_case);

   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)

   cmpl(reg, -1);

   jcc(Assembler::equal, special_case);

   // handle normal case

   bind(normal_case);

   cdql();

   int idivl_offset = offset();

   idivl(reg);

   // normal and special case exit

   bind(special_case);

   return idivl_offset;

 }

 void MacroAssembler::decrementl(Register reg, int value) {

   if (value == min_jint) {subl(reg, value) ; return; }

   if (value <  0) { incrementl(reg, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { decl(reg) ; return; }

   /* else */      { subl(reg, value)       ; return; }

 }

 void MacroAssembler::decrementl(Address dst, int value) {

   if (value == min_jint) {subl(dst, value) ; return; }

   if (value <  0) { incrementl(dst, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { decl(dst) ; return; }

   /* else */      { subl(dst, value)       ; return; }

 }

 void MacroAssembler::division_with_shift (Register reg, int shift_value) {

   assert (shift_value > 0, "illegal shift value");

   Label _is_positive;

   testl (reg, reg);

   jcc (Assembler::positive, _is_positive);

   int offset = (1 << shift_value) - 1 ;

   if (offset == 1) {

     incrementl(reg);

   } else {

     addl(reg, offset);

   }

   bind (_is_positive);

   sarl(reg, shift_value);

 }

 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {

   if (reachable(src)) {

     Assembler::divsd(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::divsd(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {

   if (reachable(src)) {

     Assembler::divss(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::divss(dst, Address(rscratch1, 0));

   }

 }

 // !defined(COMPILER2) is because of stupid core builds

 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2)

 void MacroAssembler::empty_FPU_stack() {

   if (VM_Version::supports_mmx()) {

     emms();

   } else {

     for (int i = 8; i-- > 0; ) ffree(i);

   }

 }

 #endif // !LP64 || C1 || !C2

 // Defines obj, preserves var_size_in_bytes

 void MacroAssembler::eden_allocate(Register obj,

                                    Register var_size_in_bytes,

                                    int con_size_in_bytes,

                                    Register t1,

                                    Label& slow_case) {

   assert(obj == rax, "obj must be in rax, for cmpxchg");

   assert_different_registers(obj, var_size_in_bytes, t1);

   if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {

     jmp(slow_case);

   } else {

     Register end = t1;

     Label retry;

     bind(retry);

     ExternalAddress heap_top((address) Universe::heap()->top_addr());

     movptr(obj, heap_top);

     if (var_size_in_bytes == noreg) {

       lea(end, Address(obj, con_size_in_bytes));

     } else {

       lea(end, Address(obj, var_size_in_bytes, Address::times_1));

     }

     // if end < obj then we wrapped around => object too long => slow case

     cmpptr(end, obj);

     jcc(Assembler::below, slow_case);

     cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));

     jcc(Assembler::above, slow_case);

     // Compare obj with the top addr, and if still equal, store the new top addr in

     // end at the address of the top addr pointer. Sets ZF if was equal, and clears

     // it otherwise. Use lock prefix for atomicity on MPs.

     locked_cmpxchgptr(end, heap_top);

     jcc(Assembler::notEqual, retry);

   }

 }

 void MacroAssembler::enter() {

   push(rbp);

   mov(rbp, rsp);

 }

 // A 5 byte nop that is safe for patching (see patch_verified_entry)

 void MacroAssembler::fat_nop() {

   if (UseAddressNop) {

     addr_nop_5();

   } else {

     emit_byte(0x26); // es:

     emit_byte(0x2e); // cs:

     emit_byte(0x64); // fs:

     emit_byte(0x65); // gs:

     emit_byte(0x90);

   }

 }

 void MacroAssembler::fcmp(Register tmp) {

   fcmp(tmp, 1, true, true);

 }

 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {

   assert(!pop_right || pop_left, "usage error");

   if (VM_Version::supports_cmov()) {

     assert(tmp == noreg, "unneeded temp");

     if (pop_left) {

       fucomip(index);

     } else {

       fucomi(index);

     }

     if (pop_right) {

       fpop();

     }

   } else {

     assert(tmp != noreg, "need temp");

     if (pop_left) {

       if (pop_right) {

         fcompp();

       } else {

         fcomp(index);

       }

     } else {

       fcom(index);

     }

     // convert FPU condition into eflags condition via rax,

     save_rax(tmp);

     fwait(); fnstsw_ax();

     sahf();

     restore_rax(tmp);

   }

   // condition codes set as follows:

   //

   // CF (corresponds to C0) if x < y

   // PF (corresponds to C2) if unordered

   // ZF (corresponds to C3) if x = y

 }

 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {

   fcmp2int(dst, unordered_is_less, 1, true, true);

 }

 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {

   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);

   Label L;

   if (unordered_is_less) {

     movl(dst, -1);

     jcc(Assembler::parity, L);

     jcc(Assembler::below , L);

     movl(dst, 0);

     jcc(Assembler::equal , L);

     increment(dst);

   } else { // unordered is greater

     movl(dst, 1);

     jcc(Assembler::parity, L);

     jcc(Assembler::above , L);

     movl(dst, 0);

     jcc(Assembler::equal , L);

     decrementl(dst);

   }

   bind(L);

 }

 void MacroAssembler::fld_d(AddressLiteral src) {

   fld_d(as_Address(src));

 }

 void MacroAssembler::fld_s(AddressLiteral src) {

   fld_s(as_Address(src));

 }

 void MacroAssembler::fld_x(AddressLiteral src) {

   Assembler::fld_x(as_Address(src));

 }

 void MacroAssembler::fldcw(AddressLiteral src) {

   Assembler::fldcw(as_Address(src));

 }

 void MacroAssembler::pow_exp_core_encoding() {

   // kills rax, rcx, rdx

   subptr(rsp,sizeof(jdouble));

   // computes 2^X. Stack: X ...

   // f2xm1 computes 2^X-1 but only operates on -1<=X<=1. Get int(X) and

   // keep it on the thread's stack to compute 2^int(X) later

   // then compute 2^(X-int(X)) as (2^(X-int(X)-1+1)

   // final result is obtained with: 2^X = 2^int(X) * 2^(X-int(X))

   fld_s(0);                 // Stack: X X ...

   frndint();                // Stack: int(X) X ...

   fsuba(1);                 // Stack: int(X) X-int(X) ...

   fistp_s(Address(rsp,0));  // move int(X) as integer to thread's stack. Stack: X-int(X) ...

   f2xm1();                  // Stack: 2^(X-int(X))-1 ...

   fld1();                   // Stack: 1 2^(X-int(X))-1 ...

   faddp(1);                 // Stack: 2^(X-int(X))

   // computes 2^(int(X)): add exponent bias (1023) to int(X), then

   // shift int(X)+1023 to exponent position.

   // Exponent is limited to 11 bits if int(X)+1023 does not fit in 11

   // bits, set result to NaN. 0x000 and 0x7FF are reserved exponent

   // values so detect them and set result to NaN.

   movl(rax,Address(rsp,0));

   movl(rcx, -2048); // 11 bit mask and valid NaN binary encoding

   addl(rax, 1023);

   movl(rdx,rax);

   shll(rax,20);

   // Check that 0 < int(X)+1023 < 2047. Otherwise set rax to NaN.

   addl(rdx,1);

   // Check that 1 < int(X)+1023+1 < 2048

   // in 3 steps:

   // 1- (int(X)+1023+1)&-2048 == 0 => 0 <= int(X)+1023+1 < 2048

   // 2- (int(X)+1023+1)&-2048 != 0

   // 3- (int(X)+1023+1)&-2048 != 1

   // Do 2- first because addl just updated the flags.

   cmov32(Assembler::equal,rax,rcx);

   cmpl(rdx,1);

   cmov32(Assembler::equal,rax,rcx);

   testl(rdx,rcx);

   cmov32(Assembler::notEqual,rax,rcx);

   movl(Address(rsp,4),rax);

   movl(Address(rsp,0),0);

   fmul_d(Address(rsp,0));   // Stack: 2^X ...

   addptr(rsp,sizeof(jdouble));

 }

 void MacroAssembler::increase_precision() {

   subptr(rsp, BytesPerWord);

   fnstcw(Address(rsp, 0));

   movl(rax, Address(rsp, 0));

   orl(rax, 0x300);

   push(rax);

   fldcw(Address(rsp, 0));

   pop(rax);

 }

 void MacroAssembler::restore_precision() {

   fldcw(Address(rsp, 0));

   addptr(rsp, BytesPerWord);

 }

 void MacroAssembler::fast_pow() {

   // computes X^Y = 2^(Y * log2(X))

   // if fast computation is not possible, result is NaN. Requires

   // fallback from user of this macro.

   // increase precision for intermediate steps of the computation

   increase_precision();

   fyl2x();                 // Stack: (Y*log2(X)) ...

   pow_exp_core_encoding(); // Stack: exp(X) ...

   restore_precision();

 }

 void MacroAssembler::fast_exp() {

   // computes exp(X) = 2^(X * log2(e))

   // if fast computation is not possible, result is NaN. Requires

   // fallback from user of this macro.

   // increase precision for intermediate steps of the computation

   increase_precision();

   fldl2e();                // Stack: log2(e) X ...

   fmulp(1);                // Stack: (X*log2(e)) ...

   pow_exp_core_encoding(); // Stack: exp(X) ...

   restore_precision();

 }

 void MacroAssembler::pow_or_exp(bool is_exp, int num_fpu_regs_in_use) {

   // kills rax, rcx, rdx

   // pow and exp needs 2 extra registers on the fpu stack.

   Label slow_case, done;

   Register tmp = noreg;

   if (!VM_Version::supports_cmov()) {

     // fcmp needs a temporary so preserve rdx,

     tmp = rdx;

   }

   Register tmp2 = rax;

   Register tmp3 = rcx;

   if (is_exp) {

     // Stack: X

     fld_s(0);                   // duplicate argument for runtime call. Stack: X X

     fast_exp();                 // Stack: exp(X) X

     fcmp(tmp, 0, false, false); // Stack: exp(X) X

     // exp(X) not equal to itself: exp(X) is NaN go to slow case.

     jcc(Assembler::parity, slow_case);

     // get rid of duplicate argument. Stack: exp(X)

     if (num_fpu_regs_in_use > 0) {

       fxch();

       fpop();

     } else {

       ffree(1);

     }

     jmp(done);

   } else {

     // Stack: X Y

     Label x_negative, y_odd;

     fldz();                     // Stack: 0 X Y

     fcmp(tmp, 1, true, false);  // Stack: X Y

     jcc(Assembler::above, x_negative);

     // X >= 0

     fld_s(1);                   // duplicate arguments for runtime call. Stack: Y X Y

     fld_s(1);                   // Stack: X Y X Y

     fast_pow();                 // Stack: X^Y X Y

     fcmp(tmp, 0, false, false); // Stack: X^Y X Y

     // X^Y not equal to itself: X^Y is NaN go to slow case.

     jcc(Assembler::parity, slow_case);

     // get rid of duplicate arguments. Stack: X^Y

     if (num_fpu_regs_in_use > 0) {

       fxch(); fpop();

       fxch(); fpop();

     } else {

       ffree(2);

       ffree(1);

     }

     jmp(done);

     // X <= 0

     bind(x_negative);

     fld_s(1);                   // Stack: Y X Y

     frndint();                  // Stack: int(Y) X Y

     fcmp(tmp, 2, false, false); // Stack: int(Y) X Y

     jcc(Assembler::notEqual, slow_case);

     subptr(rsp, 8);

     // For X^Y, when X < 0, Y has to be an integer and the final

     // result depends on whether it's odd or even. We just checked

     // that int(Y) == Y.  We move int(Y) to gp registers as a 64 bit

     // integer to test its parity. If int(Y) is huge and doesn't fit

     // in the 64 bit integer range, the integer indefinite value will

     // end up in the gp registers. Huge numbers are all even, the

     // integer indefinite number is even so it's fine.

 #ifdef ASSERT

     // Let's check we don't end up with an integer indefinite number

     // when not expected. First test for huge numbers: check whether

     // int(Y)+1 == int(Y) which is true for very large numbers and

     // those are all even. A 64 bit integer is guaranteed to not

     // overflow for numbers where y+1 != y (when precision is set to

     // double precision).

     Label y_not_huge;

     fld1();                     // Stack: 1 int(Y) X Y

     fadd(1);                    // Stack: 1+int(Y) int(Y) X Y

 #ifdef _LP64

     // trip to memory to force the precision down from double extended

     // precision

     fstp_d(Address(rsp, 0));

     fld_d(Address(rsp, 0));

 #endif

     fcmp(tmp, 1, true, false);  // Stack: int(Y) X Y

 #endif

     // move int(Y) as 64 bit integer to thread's stack

     fistp_d(Address(rsp,0));    // Stack: X Y

 #ifdef ASSERT

     jcc(Assembler::notEqual, y_not_huge);

     // Y is huge so we know it's even. It may not fit in a 64 bit

     // integer and we don't want the debug code below to see the

     // integer indefinite value so overwrite int(Y) on the thread's

     // stack with 0.

     movl(Address(rsp, 0), 0);

     movl(Address(rsp, 4), 0);

     bind(y_not_huge);

 #endif

     fld_s(1);                   // duplicate arguments for runtime call. Stack: Y X Y

     fld_s(1);                   // Stack: X Y X Y

     fabs();                     // Stack: abs(X) Y X Y

     fast_pow();                 // Stack: abs(X)^Y X Y

     fcmp(tmp, 0, false, false); // Stack: abs(X)^Y X Y

     // abs(X)^Y not equal to itself: abs(X)^Y is NaN go to slow case.

     pop(tmp2);

     NOT_LP64(pop(tmp3));

     jcc(Assembler::parity, slow_case);

 #ifdef ASSERT

     // Check that int(Y) is not integer indefinite value (int

     // overflow). Shouldn't happen because for values that would

     // overflow, 1+int(Y)==Y which was tested earlier.

 #ifndef _LP64

     {

       Label integer;

       testl(tmp2, tmp2);

       jcc(Assembler::notZero, integer);

       cmpl(tmp3, 0x80000000);

       jcc(Assembler::notZero, integer);

       STOP("integer indefinite value shouldn't be seen here");

       bind(integer);

     }

 #else

     {

       Label integer;

       mov(tmp3, tmp2); // preserve tmp2 for parity check below

       shlq(tmp3, 1);

       jcc(Assembler::carryClear, integer);

       jcc(Assembler::notZero, integer);

       STOP("integer indefinite value shouldn't be seen here");

       bind(integer);

     }

 #endif

 #endif

     // get rid of duplicate arguments. Stack: X^Y

     if (num_fpu_regs_in_use > 0) {

       fxch(); fpop();

       fxch(); fpop();

     } else {

       ffree(2);

       ffree(1);

     }

     testl(tmp2, 1);

     jcc(Assembler::zero, done); // X <= 0, Y even: X^Y = abs(X)^Y

     // X <= 0, Y even: X^Y = -abs(X)^Y

     fchs();                     // Stack: -abs(X)^Y Y

     jmp(done);

   }

   // slow case: runtime call

   bind(slow_case);

   fpop();                       // pop incorrect result or int(Y)

   fp_runtime_fallback(is_exp ? CAST_FROM_FN_PTR(address, SharedRuntime::dexp) : CAST_FROM_FN_PTR(address, SharedRuntime::dpow),

                       is_exp ? 1 : 2, num_fpu_regs_in_use);

   // Come here with result in F-TOS

   bind(done);

 }

 void MacroAssembler::fpop() {

   ffree();

   fincstp();

 }

 void MacroAssembler::fremr(Register tmp) {

   save_rax(tmp);

   { Label L;

     bind(L);

     fprem();

     fwait(); fnstsw_ax();

 #ifdef _LP64

     testl(rax, 0x400);

     jcc(Assembler::notEqual, L);

 #else

     sahf();

     jcc(Assembler::parity, L);

 #endif // _LP64

   }

   restore_rax(tmp);

   // Result is in ST0.

   // Note: fxch & fpop to get rid of ST1

   // (otherwise FPU stack could overflow eventually)

   fxch(1);

   fpop();

 }

 void MacroAssembler::incrementl(AddressLiteral dst) {

   if (reachable(dst)) {

     incrementl(as_Address(dst));

   } else {

     lea(rscratch1, dst);

     incrementl(Address(rscratch1, 0));

   }

 }

 void MacroAssembler::incrementl(ArrayAddress dst) {

   incrementl(as_Address(dst));

 }

 void MacroAssembler::incrementl(Register reg, int value) {

   if (value == min_jint) {addl(reg, value) ; return; }

   if (value <  0) { decrementl(reg, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { incl(reg) ; return; }

   /* else */      { addl(reg, value)       ; return; }

 }

 void MacroAssembler::incrementl(Address dst, int value) {

   if (value == min_jint) {addl(dst, value) ; return; }

   if (value <  0) { decrementl(dst, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { incl(dst) ; return; }

   /* else */      { addl(dst, value)       ; return; }

 }

 void MacroAssembler::jump(AddressLiteral dst) {

   if (reachable(dst)) {

     jmp_literal(dst.target(), dst.rspec());

   } else {

     lea(rscratch1, dst);

     jmp(rscratch1);

   }

 }

 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {

   if (reachable(dst)) {

     InstructionMark im(this);

     relocate(dst.reloc());

     const int short_size = 2;

     const int long_size = 6;

     int offs = (intptr_t)dst.target() - ((intptr_t)pc());

     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {

       // 0111 tttn #8-bit disp

       emit_byte(0x70 | cc);

       emit_byte((offs - short_size) & 0xFF);

     } else {

       // 0000 1111 1000 tttn #32-bit disp

       emit_byte(0x0F);

       emit_byte(0x80 | cc);

       emit_long(offs - long_size);

     }

   } else {

 #ifdef ASSERT

     warning("reversing conditional branch");

 #endif /* ASSERT */

     Label skip;

     jccb(reverse[cc], skip);

     lea(rscratch1, dst);

     Assembler::jmp(rscratch1);

     bind(skip);

   }

 }

 void MacroAssembler::ldmxcsr(AddressLiteral src) {

   if (reachable(src)) {

     Assembler::ldmxcsr(as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::ldmxcsr(Address(rscratch1, 0));

   }

 }

 int MacroAssembler::load_signed_byte(Register dst, Address src) {

   int off;

   if (LP64_ONLY(true ||) VM_Version::is_P6()) {

     off = offset();

     movsbl(dst, src); // movsxb

   } else {

     off = load_unsigned_byte(dst, src);

     shll(dst, 24);

     sarl(dst, 24);

   }

   return off;

 }

 // Note: load_signed_short used to be called load_signed_word.

 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler

 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.

 // The term "word" in HotSpot means a 32- or 64-bit machine word.

 int MacroAssembler::load_signed_short(Register dst, Address src) {

   int off;

   if (LP64_ONLY(true ||) VM_Version::is_P6()) {

     // This is dubious to me since it seems safe to do a signed 16 => 64 bit

     // version but this is what 64bit has always done. This seems to imply

     // that users are only using 32bits worth.

     off = offset();

     movswl(dst, src); // movsxw

   } else {

     off = load_unsigned_short(dst, src);

     shll(dst, 16);

     sarl(dst, 16);

   }

   return off;

 }

 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {

   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,

   // and "3.9 Partial Register Penalties", p. 22).

   int off;

   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {

     off = offset();

     movzbl(dst, src); // movzxb

   } else {

     xorl(dst, dst);

     off = offset();

     movb(dst, src);

   }

   return off;

 }

 // Note: load_unsigned_short used to be called load_unsigned_word.

 int MacroAssembler::load_unsigned_short(Register dst, Address src) {

   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,

   // and "3.9 Partial Register Penalties", p. 22).

   int off;

   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {

     off = offset();

     movzwl(dst, src); // movzxw

   } else {

     xorl(dst, dst);

     off = offset();

     movw(dst, src);

   }

   return off;

 }

 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {

   switch (size_in_bytes) {

 #ifndef _LP64

   case  8:

     assert(dst2 != noreg, "second dest register required");

     movl(dst,  src);

     movl(dst2, src.plus_disp(BytesPerInt));

     break;

 #else

   case  8:  movq(dst, src); break;

 #endif

   case  4:  movl(dst, src); break;

   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;

   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;

   default:  ShouldNotReachHere();

   }