Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "precompiled.hpp"

 #include "asm/assembler.hpp"

 #include "asm/assembler.inline.hpp"

 #include "compiler/disassembler.hpp"

 #include "gc_interface/collectedHeap.inline.hpp"

 #include "interpreter/interpreter.hpp"

 #include "memory/cardTableModRefBS.hpp"

 #include "memory/resourceArea.hpp"

 #include "memory/universe.hpp"

 #include "prims/methodHandles.hpp"

 #include "runtime/biasedLocking.hpp"

 #include "runtime/interfaceSupport.hpp"

 #include "runtime/objectMonitor.hpp"

 #include "runtime/os.hpp"

 #include "runtime/sharedRuntime.hpp"

 #include "runtime/stubRoutines.hpp"

 #include "utilities/macros.hpp"

 #if INCLUDE_ALL_GCS

 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"

 #include "gc_implementation/g1/heapRegion.hpp"

 #endif // INCLUDE_ALL_GCS

 #ifdef PRODUCT

 #define BLOCK_COMMENT(str) /* nothing */

 #define STOP(error) stop(error)

 #else

 #define BLOCK_COMMENT(str) block_comment(str)

 #define STOP(error) block_comment(error); stop(error)

 #endif

 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

 #ifdef ASSERT

 bool AbstractAssembler::pd_check_instruction_mark() { return true; }

 #endif

 static Assembler::Condition reverse[] = {

     Assembler::noOverflow     /* overflow      = 0x0 */ ,

     Assembler::overflow       /* noOverflow    = 0x1 */ ,

     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,

     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,

     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,

     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,

     Assembler::above          /* belowEqual    = 0x6 */ ,

     Assembler::belowEqual     /* above         = 0x7 */ ,

     Assembler::positive       /* negative      = 0x8 */ ,

     Assembler::negative       /* positive      = 0x9 */ ,

     Assembler::noParity       /* parity        = 0xa */ ,

     Assembler::parity         /* noParity      = 0xb */ ,

     Assembler::greaterEqual   /* less          = 0xc */ ,

     Assembler::less           /* greaterEqual  = 0xd */ ,

     Assembler::greater        /* lessEqual     = 0xe */ ,

     Assembler::lessEqual      /* greater       = 0xf, */

 };

 // Implementation of MacroAssembler

 // First all the versions that have distinct versions depending on 32/64 bit

 // Unless the difference is trivial (1 line or so).

 #ifndef _LP64

 // 32bit versions

 Address MacroAssembler::as_Address(AddressLiteral adr) {

   return Address(adr.target(), adr.rspec());

 }

 Address MacroAssembler::as_Address(ArrayAddress adr) {

   return Address::make_array(adr);

 }

 void MacroAssembler::call_VM_leaf_base(address entry_point,

                                        int number_of_arguments) {

   call(RuntimeAddress(entry_point));

   increment(rsp, number_of_arguments * wordSize);

 }

 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {

   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {

   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::cmpoop(Address src1, jobject obj) {

   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::cmpoop(Register src1, jobject obj) {

   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::extend_sign(Register hi, Register lo) {

   // According to Intel Doc. AP-526, "Integer Divide", p.18.

   if (VM_Version::is_P6() && hi == rdx && lo == rax) {

     cdql();

   } else {

     movl(hi, lo);

     sarl(hi, 31);

   }

 }

 void MacroAssembler::jC2(Register tmp, Label& L) {

   // set parity bit if FPU flag C2 is set (via rax)

   save_rax(tmp);

   fwait(); fnstsw_ax();

   sahf();

   restore_rax(tmp);

   // branch

   jcc(Assembler::parity, L);

 }

 void MacroAssembler::jnC2(Register tmp, Label& L) {

   // set parity bit if FPU flag C2 is set (via rax)

   save_rax(tmp);

   fwait(); fnstsw_ax();

   sahf();

   restore_rax(tmp);

   // branch

   jcc(Assembler::noParity, L);

 }

 // 32bit can do a case table jump in one instruction but we no longer allow the base

 // to be installed in the Address class

 void MacroAssembler::jump(ArrayAddress entry) {

   jmp(as_Address(entry));

 }

 // Note: y_lo will be destroyed

 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {

   // Long compare for Java (semantics as described in JVM spec.)

   Label high, low, done;

   cmpl(x_hi, y_hi);

   jcc(Assembler::less, low);

   jcc(Assembler::greater, high);

   // x_hi is the return register

   xorl(x_hi, x_hi);

   cmpl(x_lo, y_lo);

   jcc(Assembler::below, low);

   jcc(Assembler::equal, done);

   bind(high);

   xorl(x_hi, x_hi);

   increment(x_hi);

   jmp(done);

   bind(low);

   xorl(x_hi, x_hi);

   decrementl(x_hi);

   bind(done);

 }

 void MacroAssembler::lea(Register dst, AddressLiteral src) {

     mov_literal32(dst, (int32_t)src.target(), src.rspec());

 }

 void MacroAssembler::lea(Address dst, AddressLiteral adr) {

   // leal(dst, as_Address(adr));

   // see note in movl as to why we must use a move

   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());

 }

 void MacroAssembler::leave() {

   mov(rsp, rbp);

   pop(rbp);

 }

 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {

   // Multiplication of two Java long values stored on the stack

   // as illustrated below. Result is in rdx:rax.

   //

   // rsp ---> [  ??  ] \               \

   //            ....    | y_rsp_offset  |

   //          [ y_lo ] /  (in bytes)    | x_rsp_offset

   //          [ y_hi ]                  | (in bytes)

   //            ....                    |

   //          [ x_lo ]                 /

   //          [ x_hi ]

   //            ....

   //

   // Basic idea: lo(result) = lo(x_lo * y_lo)

   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)

   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);

   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);

   Label quick;

   // load x_hi, y_hi and check if quick

   // multiplication is possible

   movl(rbx, x_hi);

   movl(rcx, y_hi);

   movl(rax, rbx);

   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0

   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply

   // do full multiplication

   // 1st step

   mull(y_lo);                                    // x_hi * y_lo

   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,

   // 2nd step

   movl(rax, x_lo);

   mull(rcx);                                     // x_lo * y_hi

   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,

   // 3rd step

   bind(quick);                                   // note: rbx, = 0 if quick multiply!

   movl(rax, x_lo);

   mull(y_lo);                                    // x_lo * y_lo

   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)

 }

 void MacroAssembler::lneg(Register hi, Register lo) {

   negl(lo);

   adcl(hi, 0);

   negl(hi);

 }

 void MacroAssembler::lshl(Register hi, Register lo) {

   // Java shift left long support (semantics as described in JVM spec., p.305)

   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))

   // shift value is in rcx !

   assert(hi != rcx, "must not use rcx");

   assert(lo != rcx, "must not use rcx");

   const Register s = rcx;                        // shift count

   const int      n = BitsPerWord;

   Label L;

   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)

   cmpl(s, n);                                    // if (s < n)

   jcc(Assembler::less, L);                       // else (s >= n)

   movl(hi, lo);                                  // x := x << n

   xorl(lo, lo);

   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!

   bind(L);                                       // s (mod n) < n

   shldl(hi, lo);                                 // x := x << s

   shll(lo);

 }

 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {

   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)

   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))

   assert(hi != rcx, "must not use rcx");

   assert(lo != rcx, "must not use rcx");

   const Register s = rcx;                        // shift count

   const int      n = BitsPerWord;

   Label L;

   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)

   cmpl(s, n);                                    // if (s < n)

   jcc(Assembler::less, L);                       // else (s >= n)

   movl(lo, hi);                                  // x := x >> n

   if (sign_extension) sarl(hi, 31);

   else                xorl(hi, hi);

   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!

   bind(L);                                       // s (mod n) < n

   shrdl(lo, hi);                                 // x := x >> s

   if (sign_extension) sarl(hi);

   else                shrl(hi);

 }

 void MacroAssembler::movoop(Register dst, jobject obj) {

   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::movoop(Address dst, jobject obj) {

   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {

   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {

   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {

   // scratch register is not used,

   // it is defined to match parameters of 64-bit version of this method.

   if (src.is_lval()) {

     mov_literal32(dst, (intptr_t)src.target(), src.rspec());

   } else {

     movl(dst, as_Address(src));

   }

 }

 void MacroAssembler::movptr(ArrayAddress dst, Register src) {

   movl(as_Address(dst), src);

 }

 void MacroAssembler::movptr(Register dst, ArrayAddress src) {

   movl(dst, as_Address(src));

 }

 // src should NEVER be a real pointer. Use AddressLiteral for true pointers

 void MacroAssembler::movptr(Address dst, intptr_t src) {

   movl(dst, src);

 }

 void MacroAssembler::pop_callee_saved_registers() {

   pop(rcx);

   pop(rdx);

   pop(rdi);

   pop(rsi);

 }

 void MacroAssembler::pop_fTOS() {

   fld_d(Address(rsp, 0));

   addl(rsp, 2 * wordSize);

 }

 void MacroAssembler::push_callee_saved_registers() {

   push(rsi);

   push(rdi);

   push(rdx);

   push(rcx);

 }

 void MacroAssembler::push_fTOS() {

   subl(rsp, 2 * wordSize);

   fstp_d(Address(rsp, 0));

 }

 void MacroAssembler::pushoop(jobject obj) {

   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::pushklass(Metadata* obj) {

   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::pushptr(AddressLiteral src) {

   if (src.is_lval()) {

     push_literal32((int32_t)src.target(), src.rspec());

   } else {

     pushl(as_Address(src));

   }

 }

 void MacroAssembler::set_word_if_not_zero(Register dst) {

   xorl(dst, dst);

   set_byte_if_not_zero(dst);

 }

 static void pass_arg0(MacroAssembler* masm, Register arg) {

   masm->push(arg);

 }

 static void pass_arg1(MacroAssembler* masm, Register arg) {

   masm->push(arg);

 }

 static void pass_arg2(MacroAssembler* masm, Register arg) {

   masm->push(arg);

 }

 static void pass_arg3(MacroAssembler* masm, Register arg) {

   masm->push(arg);

 }

 #ifndef PRODUCT

 extern "C" void findpc(intptr_t x);

 #endif

 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {

   // In order to get locks to work, we need to fake a in_VM state

   JavaThread* thread = JavaThread::current();

   JavaThreadState saved_state = thread->thread_state();

   thread->set_thread_state(_thread_in_vm);

   if (ShowMessageBoxOnError) {

     JavaThread* thread = JavaThread::current();

     JavaThreadState saved_state = thread->thread_state();

     thread->set_thread_state(_thread_in_vm);

     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {

       ttyLocker ttyl;

       BytecodeCounter::print();

     }

     // To see where a verify_oop failed, get $ebx+40/X for this frame.

     // This is the value of eip which points to where verify_oop will return.

     if (os::message_box(msg, "Execution stopped, print registers?")) {

       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);

       BREAKPOINT;

     }

   } else {

     ttyLocker ttyl;

     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);

   }

   // Don't assert holding the ttyLock

     assert(false, err_msg("DEBUG MESSAGE: %s", msg));

   ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);

 }

 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {

   ttyLocker ttyl;

   FlagSetting fs(Debugging, true);

   tty->print_cr("eip = 0x%08x", eip);

 #ifndef PRODUCT

   if ((WizardMode || Verbose) && PrintMiscellaneous) {

     tty->cr();

     findpc(eip);

     tty->cr();

   }

 #endif

 #define PRINT_REG(rax) \

   { tty->print("%s = ", #rax); os::print_location(tty, rax); }

   PRINT_REG(rax);

   PRINT_REG(rbx);

   PRINT_REG(rcx);

   PRINT_REG(rdx);

   PRINT_REG(rdi);

   PRINT_REG(rsi);

   PRINT_REG(rbp);

   PRINT_REG(rsp);

 #undef PRINT_REG

   // Print some words near top of staack.

   int* dump_sp = (int*) rsp;

   for (int col1 = 0; col1 < 8; col1++) {

     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);

     os::print_location(tty, *dump_sp++);

   }

   for (int row = 0; row < 16; row++) {

     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);

     for (int col = 0; col < 8; col++) {

       tty->print(" 0x%08x", *dump_sp++);

     }

     tty->cr();

   }

   // Print some instructions around pc:

   Disassembler::decode((address)eip-64, (address)eip);

   tty->print_cr("--------");

   Disassembler::decode((address)eip, (address)eip+32);

 }

 void MacroAssembler::stop(const char* msg) {

   ExternalAddress message((address)msg);

   // push address of message

   pushptr(message.addr());

   { Label L; call(L, relocInfo::none); bind(L); }     // push eip

   pusha();                                            // push registers

   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));

   hlt();

 }

 void MacroAssembler::warn(const char* msg) {

   push_CPU_state();

   ExternalAddress message((address) msg);

   // push address of message

   pushptr(message.addr());

   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));

   addl(rsp, wordSize);       // discard argument

   pop_CPU_state();

 }

 void MacroAssembler::print_state() {

   { Label L; call(L, relocInfo::none); bind(L); }     // push eip

   pusha();                                            // push registers

   push_CPU_state();

   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));

   pop_CPU_state();

   popa();

   addl(rsp, wordSize);

 }

 #else // _LP64

 // 64 bit versions

 Address MacroAssembler::as_Address(AddressLiteral adr) {

   // amd64 always does this as a pc-rel

   // we can be absolute or disp based on the instruction type

   // jmp/call are displacements others are absolute

   assert(!adr.is_lval(), "must be rval");

   assert(reachable(adr), "must be");

   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());

 }

 Address MacroAssembler::as_Address(ArrayAddress adr) {

   AddressLiteral base = adr.base();

   lea(rscratch1, base);

   Address index = adr.index();

   assert(index._disp == 0, "must not have disp"); // maybe it can?

   Address array(rscratch1, index._index, index._scale, index._disp);

   return array;

 }

 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {

   Label L, E;

 #ifdef _WIN64

   // Windows always allocates space for it's register args

   assert(num_args <= 4, "only register arguments supported");

   subq(rsp,  frame::arg_reg_save_area_bytes);

 #endif

   // Align stack if necessary

   testl(rsp, 15);

   jcc(Assembler::zero, L);

   subq(rsp, 8);

   {

     call(RuntimeAddress(entry_point));

   }

   addq(rsp, 8);

   jmp(E);

   bind(L);

   {

     call(RuntimeAddress(entry_point));

   }

   bind(E);

 #ifdef _WIN64

   // restore stack pointer

   addq(rsp, frame::arg_reg_save_area_bytes);

 #endif

 }

 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {

   assert(!src2.is_lval(), "should use cmpptr");

   if (reachable(src2)) {

     cmpq(src1, as_Address(src2));

   } else {

     lea(rscratch1, src2);

     Assembler::cmpq(src1, Address(rscratch1, 0));

   }

 }

 int MacroAssembler::corrected_idivq(Register reg) {

   // Full implementation of Java ldiv and lrem; checks for special

   // case as described in JVM spec., p.243 & p.271.  The function

   // returns the (pc) offset of the idivl instruction - may be needed

   // for implicit exceptions.

   //

   //         normal case                           special case

   //

   // input : rax: dividend                         min_long

   //         reg: divisor   (may not be eax/edx)   -1

   //

   // output: rax: quotient  (= rax idiv reg)       min_long

   //         rdx: remainder (= rax irem reg)       0

   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");

   static const int64_t min_long = 0x8000000000000000;

   Label normal_case, special_case;

   // check for special case

   cmp64(rax, ExternalAddress((address) &min_long));

   jcc(Assembler::notEqual, normal_case);

   xorl(rdx, rdx); // prepare rdx for possible special case (where

                   // remainder = 0)

   cmpq(reg, -1);

   jcc(Assembler::equal, special_case);

   // handle normal case

   bind(normal_case);

   cdqq();

   int idivq_offset = offset();

   idivq(reg);

   // normal and special case exit

   bind(special_case);

   return idivq_offset;

 }

 void MacroAssembler::decrementq(Register reg, int value) {

   if (value == min_jint) { subq(reg, value); return; }

   if (value <  0) { incrementq(reg, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { decq(reg) ; return; }

   /* else */      { subq(reg, value)       ; return; }

 }

 void MacroAssembler::decrementq(Address dst, int value) {

   if (value == min_jint) { subq(dst, value); return; }

   if (value <  0) { incrementq(dst, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { decq(dst) ; return; }

   /* else */      { subq(dst, value)       ; return; }

 }

 void MacroAssembler::incrementq(AddressLiteral dst) {

   if (reachable(dst)) {

     incrementq(as_Address(dst));

   } else {

     lea(rscratch1, dst);

     incrementq(Address(rscratch1, 0));

   }

 }

 void MacroAssembler::incrementq(Register reg, int value) {

   if (value == min_jint) { addq(reg, value); return; }

   if (value <  0) { decrementq(reg, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { incq(reg) ; return; }

   /* else */      { addq(reg, value)       ; return; }

 }

 void MacroAssembler::incrementq(Address dst, int value) {

   if (value == min_jint) { addq(dst, value); return; }

   if (value <  0) { decrementq(dst, -value); return; }

   if (value == 0) {                        ; return; }

   if (value == 1 && UseIncDec) { incq(dst) ; return; }

   /* else */      { addq(dst, value)       ; return; }

 }

 // 32bit can do a case table jump in one instruction but we no longer allow the base

 // to be installed in the Address class

 void MacroAssembler::jump(ArrayAddress entry) {

   lea(rscratch1, entry.base());

   Address dispatch = entry.index();

   assert(dispatch._base == noreg, "must be");

   dispatch._base = rscratch1;

   jmp(dispatch);

 }

 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {

   ShouldNotReachHere(); // 64bit doesn't use two regs

   cmpq(x_lo, y_lo);

 }

 void MacroAssembler::lea(Register dst, AddressLiteral src) {

     mov_literal64(dst, (intptr_t)src.target(), src.rspec());

 }

 void MacroAssembler::lea(Address dst, AddressLiteral adr) {

   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());

   movptr(dst, rscratch1);

 }

 void MacroAssembler::leave() {

   // %%% is this really better? Why not on 32bit too?

   emit_int8((unsigned char)0xC9); // LEAVE

 }

 void MacroAssembler::lneg(Register hi, Register lo) {

   ShouldNotReachHere(); // 64bit doesn't use two regs

   negq(lo);

 }

 void MacroAssembler::movoop(Register dst, jobject obj) {

   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());

 }

 void MacroAssembler::movoop(Address dst, jobject obj) {

   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());

   movq(dst, rscratch1);

 }

 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {

   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());

 }

 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {

   mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());

   movq(dst, rscratch1);

 }

 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {

   if (src.is_lval()) {

     mov_literal64(dst, (intptr_t)src.target(), src.rspec());

   } else {

     if (reachable(src)) {

       movq(dst, as_Address(src));

     } else {

       lea(scratch, src);

       movq(dst, Address(scratch, 0));

     }

   }

 }

 void MacroAssembler::movptr(ArrayAddress dst, Register src) {

   movq(as_Address(dst), src);

 }

 void MacroAssembler::movptr(Register dst, ArrayAddress src) {

   movq(dst, as_Address(src));

 }

 // src should NEVER be a real pointer. Use AddressLiteral for true pointers

 void MacroAssembler::movptr(Address dst, intptr_t src) {

   mov64(rscratch1, src);

   movq(dst, rscratch1);

 }

 // These are mostly for initializing NULL

 void MacroAssembler::movptr(Address dst, int32_t src) {

   movslq(dst, src);

 }

 void MacroAssembler::movptr(Register dst, int32_t src) {

   mov64(dst, (intptr_t)src);

 }

 void MacroAssembler::pushoop(jobject obj) {

   movoop(rscratch1, obj);

   push(rscratch1);

 }

 void MacroAssembler::pushklass(Metadata* obj) {

   mov_metadata(rscratch1, obj);

   push(rscratch1);

 }

 void MacroAssembler::pushptr(AddressLiteral src) {

   lea(rscratch1, src);

   if (src.is_lval()) {

     push(rscratch1);

   } else {

     pushq(Address(rscratch1, 0));

   }

 }

 void MacroAssembler::reset_last_Java_frame(bool clear_fp,

                                            bool clear_pc) {

   // we must set sp to zero to clear frame

   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);

   // must clear fp, so that compiled frames are not confused; it is

   // possible that we need it only for debugging

   if (clear_fp) {

     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);

   }

   if (clear_pc) {

     movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);

   }

 }

 void MacroAssembler::set_last_Java_frame(Register last_java_sp,

                                          Register last_java_fp,

                                          address  last_java_pc) {

   // determine last_java_sp register

   if (!last_java_sp->is_valid()) {

     last_java_sp = rsp;

   }

   // last_java_fp is optional

   if (last_java_fp->is_valid()) {

     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),

            last_java_fp);

   }

   // last_java_pc is optional

   if (last_java_pc != NULL) {

     Address java_pc(r15_thread,

                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());

     lea(rscratch1, InternalAddress(last_java_pc));

     movptr(java_pc, rscratch1);

   }

   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);

 }

 static void pass_arg0(MacroAssembler* masm, Register arg) {

   if (c_rarg0 != arg ) {

     masm->mov(c_rarg0, arg);

   }

 }

 static void pass_arg1(MacroAssembler* masm, Register arg) {

   if (c_rarg1 != arg ) {

     masm->mov(c_rarg1, arg);

   }

 }

 static void pass_arg2(MacroAssembler* masm, Register arg) {

   if (c_rarg2 != arg ) {

     masm->mov(c_rarg2, arg);

   }

 }

 static void pass_arg3(MacroAssembler* masm, Register arg) {

   if (c_rarg3 != arg ) {

     masm->mov(c_rarg3, arg);

   }

 }

 void MacroAssembler::stop(const char* msg) {

   address rip = pc();

   pusha(); // get regs on stack

   lea(c_rarg0, ExternalAddress((address) msg));

   lea(c_rarg1, InternalAddress(rip));

   movq(c_rarg2, rsp); // pass pointer to regs array

   andq(rsp, -16); // align stack as required by ABI

   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));

   hlt();

 }

 void MacroAssembler::warn(const char* msg) {

   push(rbp);

   movq(rbp, rsp);

   andq(rsp, -16);     // align stack as required by push_CPU_state and call

   push_CPU_state();   // keeps alignment at 16 bytes

   lea(c_rarg0, ExternalAddress((address) msg));

   call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0);

   pop_CPU_state();

   mov(rsp, rbp);

   pop(rbp);

 }

 void MacroAssembler::print_state() {

   address rip = pc();

   pusha();            // get regs on stack

   push(rbp);

   movq(rbp, rsp);

   andq(rsp, -16);     // align stack as required by push_CPU_state and call

   push_CPU_state();   // keeps alignment at 16 bytes

   lea(c_rarg0, InternalAddress(rip));

   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array

   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);

   pop_CPU_state();

   mov(rsp, rbp);

   pop(rbp);

   popa();

 }

 #ifndef PRODUCT

 extern "C" void findpc(intptr_t x);

 #endif

 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {

   // In order to get locks to work, we need to fake a in_VM state

   if (ShowMessageBoxOnError) {

     JavaThread* thread = JavaThread::current();

     JavaThreadState saved_state = thread->thread_state();

     thread->set_thread_state(_thread_in_vm);

 #ifndef PRODUCT

     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {

       ttyLocker ttyl;

       BytecodeCounter::print();

     }

 #endif

     // To see where a verify_oop failed, get $ebx+40/X for this frame.

     // XXX correct this offset for amd64

     // This is the value of eip which points to where verify_oop will return.

     if (os::message_box(msg, "Execution stopped, print registers?")) {

       print_state64(pc, regs);

       BREAKPOINT;

       assert(false, "start up GDB");

     }

     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);

   } else {

     ttyLocker ttyl;

     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",

                     msg);

     assert(false, err_msg("DEBUG MESSAGE: %s", msg));

   }

 }

 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {

   ttyLocker ttyl;

   FlagSetting fs(Debugging, true);

   tty->print_cr("rip = 0x%016lx", pc);

 #ifndef PRODUCT

   tty->cr();

   findpc(pc);

   tty->cr();

 #endif

 #define PRINT_REG(rax, value) \

   { tty->print("%s = ", #rax); os::print_location(tty, value); }

   PRINT_REG(rax, regs[15]);

   PRINT_REG(rbx, regs[12]);

   PRINT_REG(rcx, regs[14]);

   PRINT_REG(rdx, regs[13]);

   PRINT_REG(rdi, regs[8]);

   PRINT_REG(rsi, regs[9]);

   PRINT_REG(rbp, regs[10]);

   PRINT_REG(rsp, regs[11]);

   PRINT_REG(r8 , regs[7]);

   PRINT_REG(r9 , regs[6]);

   PRINT_REG(r10, regs[5]);

   PRINT_REG(r11, regs[4]);

   PRINT_REG(r12, regs[3]);

   PRINT_REG(r13, regs[2]);

   PRINT_REG(r14, regs[1]);

   PRINT_REG(r15, regs[0]);

 #undef PRINT_REG

   // Print some words near top of staack.

   int64_t* rsp = (int64_t*) regs[11];

   int64_t* dump_sp = rsp;

   for (int col1 = 0; col1 < 8; col1++) {

     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp);

     os::print_location(tty, *dump_sp++);

   }

   for (int row = 0; row < 25; row++) {

     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp);

     for (int col = 0; col < 4; col++) {

       tty->print(" 0x%016lx", *dump_sp++);

     }

     tty->cr();

   }

   // Print some instructions around pc:

   Disassembler::decode((address)pc-64, (address)pc);

   tty->print_cr("--------");

   Disassembler::decode((address)pc, (address)pc+32);

 }

 #endif // _LP64

 // Now versions that are common to 32/64 bit

 void MacroAssembler::addptr(Register dst, int32_t imm32) {

   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));

 }

 void MacroAssembler::addptr(Register dst, Register src) {

   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));

 }

 void MacroAssembler::addptr(Address dst, Register src) {

   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));

 }

 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {

   if (reachable(src)) {

     Assembler::addsd(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::addsd(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {

   if (reachable(src)) {

     addss(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     addss(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::align(int modulus) {

   if (offset() % modulus != 0) {

     nop(modulus - (offset() % modulus));

   }

 }

 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {

   // Used in sign-masking with aligned address.

   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");

   if (reachable(src)) {

     Assembler::andpd(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::andpd(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {

   // Used in sign-masking with aligned address.

   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");

   if (reachable(src)) {

     Assembler::andps(dst, as_Address(src));

   } else {

     lea(rscratch1, src);

     Assembler::andps(dst, Address(rscratch1, 0));

   }

 }

 void MacroAssembler::andptr(Register dst, int32_t imm32) {

   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));

 }

 void MacroAssembler::atomic_incl(Address counter_addr) {

   if (os::is_MP())

     lock();

   incrementl(counter_addr);

 }

 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {

   if (reachable(counter_addr)) {

     atomic_incl(as_Address(counter_addr));

   } else {

     lea(scr, counter_addr);

     atomic_incl(Address(scr, 0));

   }

 }

 #ifdef _LP64

 void MacroAssembler::atomic_incq(Address counter_addr) {

   if (os::is_MP())

     lock();

   incrementq(counter_addr);

 }

 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {

   if (reachable(counter_addr)) {

     atomic_incq(as_Address(counter_addr));

   } else {

     lea(scr, counter_addr);

     atomic_incq(Address(scr, 0));

   }

 }

 #endif

 // Writes to stack successive pages until offset reached to check for

 // stack overflow + shadow pages.  This clobbers tmp.

 void MacroAssembler::bang_stack_size(Register size, Register tmp) {

   movptr(tmp, rsp);

   // Bang stack for total size given plus shadow page size.

   // Bang one page at a time because large size can bang beyond yellow and

   // red zones.

   Label loop;

   bind(loop);

   movl(Address(tmp, (-os::vm_page_size())), size );

   subptr(tmp, os::vm_page_size());

   subl(size, os::vm_page_size());

   jcc(Assembler::greater, loop);

   // Bang down shadow pages too.

   // At this point, (tmp-0) is the last address touched, so don't

   // touch it again.  (It was touched as (tmp-pagesize) but then tmp

   // was post-decremented.)  Skip this address by starting at i=1, and

   // touch a few more pages below.  N.B.  It is important to touch all

   // the way down to and including i=StackShadowPages.

   for (int i = 1; i <= StackShadowPages; i++) {

     // this could be any sized move but this is can be a debugging crumb

     // so the bigger the better.

     movptr(Address(tmp, (-i*os::vm_page_size())), size );

   }

 }

 int MacroAssembler::biased_locking_enter(Register lock_reg,

                                          Register obj_reg,

                                          Register swap_reg,

                                          Register tmp_reg,

                                          bool swap_reg_contains_mark,

                                          Label& done,

                                          Label* slow_case,

                                          BiasedLockingCounters* counters) {

   assert(UseBiasedLocking, "why call this otherwise?");

   assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");

   LP64_ONLY( assert(tmp_reg != noreg, "tmp_reg must be supplied"); )

   bool need_tmp_reg = false;

   if (tmp_reg == noreg) {

     need_tmp_reg = true;

     tmp_reg = lock_reg;

     assert_different_registers(lock_reg, obj_reg, swap_reg);

   } else {

     assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);

   }

   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");

   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());

   Address saved_mark_addr(lock_reg, 0);

   if (PrintBiasedLockingStatistics && counters == NULL) {

     counters = BiasedLocking::counters();

   }

   // Biased locking

   // See whether the lock is currently biased toward our thread and

   // whether the epoch is still valid

   // Note that the runtime guarantees sufficient alignment of JavaThread

   // pointers to allow age to be placed into low bits

   // First check to see whether biasing is even enabled for this object

   Label cas_label;

   int null_check_offset = -1;

   if (!swap_reg_contains_mark) {

     null_check_offset = offset();

     movptr(swap_reg, mark_addr);

   }

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   movptr(tmp_reg, swap_reg);

   andptr(tmp_reg, markOopDesc::biased_lock_mask_in_place);

   cmpptr(tmp_reg, markOopDesc::biased_lock_pattern);

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   jcc(Assembler::notEqual, cas_label);

   // The bias pattern is present in the object's header. Need to check

   // whether the bias owner and the epoch are both still current.

 #ifndef _LP64

   // Note that because there is no current thread register on x86_32 we

   // need to store off the mark word we read out of the object to

   // avoid reloading it and needing to recheck invariants below. This

   // store is unfortunate but it makes the overall code shorter and

   // simpler.

   movptr(saved_mark_addr, swap_reg);

 #endif

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   if (swap_reg_contains_mark) {

     null_check_offset = offset();

   }

   load_prototype_header(tmp_reg, obj_reg);

 #ifdef _LP64

   orptr(tmp_reg, r15_thread);

   xorptr(tmp_reg, swap_reg);

   Register header_reg = tmp_reg;

 #else

   xorptr(tmp_reg, swap_reg);

   get_thread(swap_reg);

   xorptr(swap_reg, tmp_reg);

   Register header_reg = swap_reg;

 #endif

   andptr(header_reg, ~((int) markOopDesc::age_mask_in_place));

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address) counters->biased_lock_entry_count_addr()));

   }

   jcc(Assembler::equal, done);

   Label try_revoke_bias;

   Label try_rebias;

   // At this point we know that the header has the bias pattern and

   // that we are not the bias owner in the current epoch. We need to

   // figure out more details about the state of the header in order to

   // know what operations can be legally performed on the object's

   // header.

   // If the low three bits in the xor result aren't clear, that means

   // the prototype header is no longer biased and we have to revoke

   // the bias on this object.

   testptr(header_reg, markOopDesc::biased_lock_mask_in_place);

   jccb(Assembler::notZero, try_revoke_bias);

   // Biasing is still enabled for this data type. See whether the

   // epoch of the current bias is still valid, meaning that the epoch

   // bits of the mark word are equal to the epoch bits of the

   // prototype header. (Note that the prototype header's epoch bits

   // only change at a safepoint.) If not, attempt to rebias the object

   // toward the current thread. Note that we must be absolutely sure

   // that the current epoch is invalid in order to do this because

   // otherwise the manipulations it performs on the mark word are

   // illegal.

   testptr(header_reg, markOopDesc::epoch_mask_in_place);

   jccb(Assembler::notZero, try_rebias);

   // The epoch of the current bias is still valid but we know nothing

   // about the owner; it might be set or it might be clear. Try to

   // acquire the bias of the object using an atomic operation. If this

   // fails we will go in to the runtime to revoke the object's bias.

   // Note that we first construct the presumed unbiased header so we

   // don't accidentally blow away another thread's valid bias.

   NOT_LP64( movptr(swap_reg, saved_mark_addr); )

   andptr(swap_reg,

          markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);

   if (need_tmp_reg) {

     push(tmp_reg);

   }

 #ifdef _LP64

   movptr(tmp_reg, swap_reg);

   orptr(tmp_reg, r15_thread);

 #else

   get_thread(tmp_reg);

   orptr(tmp_reg, swap_reg);

 #endif

   if (os::is_MP()) {

     lock();

   }

   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   // If the biasing toward our thread failed, this means that

   // another thread succeeded in biasing it toward itself and we

   // need to revoke that bias. The revocation will occur in the

   // interpreter runtime in the slow case.

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));

   }

   if (slow_case != NULL) {

     jcc(Assembler::notZero, *slow_case);

   }

   jmp(done);

   bind(try_rebias);

   // At this point we know the epoch has expired, meaning that the

   // current "bias owner", if any, is actually invalid. Under these

   // circumstances _only_, we are allowed to use the current header's

   // value as the comparison value when doing the cas to acquire the

   // bias in the current epoch. In other words, we allow transfer of

   // the bias from one thread to another directly in this situation.

   //

   // FIXME: due to a lack of registers we currently blow away the age

   // bits in this situation. Should attempt to preserve them.

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   load_prototype_header(tmp_reg, obj_reg);

 #ifdef _LP64

   orptr(tmp_reg, r15_thread);

 #else

   get_thread(swap_reg);

   orptr(tmp_reg, swap_reg);

   movptr(swap_reg, saved_mark_addr);

 #endif

   if (os::is_MP()) {

     lock();

   }

   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   // If the biasing toward our thread failed, then another thread

   // succeeded in biasing it toward itself and we need to revoke that

   // bias. The revocation will occur in the runtime in the slow case.

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));

   }

   if (slow_case != NULL) {

     jcc(Assembler::notZero, *slow_case);

   }

   jmp(done);

   bind(try_revoke_bias);

   // The prototype mark in the klass doesn't have the bias bit set any

   // more, indicating that objects of this data type are not supposed

   // to be biased any more. We are going to try to reset the mark of

   // this object to the prototype value and fall through to the

   // CAS-based locking scheme. Note that if our CAS fails, it means

   // that another thread raced us for the privilege of revoking the

   // bias of this particular object, so it's okay to continue in the

   // normal locking code.

   //

   // FIXME: due to a lack of registers we currently blow away the age

   // bits in this situation. Should attempt to preserve them.

   NOT_LP64( movptr(swap_reg, saved_mark_addr); )

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   load_prototype_header(tmp_reg, obj_reg);

   if (os::is_MP()) {

     lock();

   }

   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   // Fall through to the normal CAS-based lock, because no matter what

   // the result of the above CAS, some thread must have succeeded in

   // removing the bias bit from the object's header.

   if (counters != NULL) {

     cond_inc32(Assembler::zero,

                ExternalAddress((address) counters->revoked_lock_entry_count_addr()));

   }

   bind(cas_label);

   return null_check_offset;

 }

 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {

   assert(UseBiasedLocking, "why call this otherwise?");

   // Check for biased locking unlock case, which is a no-op

   // Note: we do not have to check the thread ID for two reasons.

   // First, the interpreter checks for IllegalMonitorStateException at

   // a higher level. Second, if the bias was revoked while we held the

   // lock, the object could not be rebiased toward another thread, so

   // the bias bit would be clear.

   movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));

   andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);

   cmpptr(temp_reg, markOopDesc::biased_lock_pattern);

   jcc(Assembler::equal, done);

 }

 #ifdef COMPILER2

 #if INCLUDE_RTM_OPT

 // Update rtm_counters based on abort status

 // input: abort_status

 //        rtm_counters (RTMLockingCounters*)

 // flags are killed

 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) {

   atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset()));

   if (PrintPreciseRTMLockingStatistics) {

     for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {

       Label check_abort;

       testl(abort_status, (1<<i));

       jccb(Assembler::equal, check_abort);

       atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx))));

       bind(check_abort);

     }

   }

 }

 // Branch if (random & (count-1) != 0), count is 2^n

 // tmp, scr and flags are killed

 void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) {

   assert(tmp == rax, "");

   assert(scr == rdx, "");

   rdtsc(); // modifies EDX:EAX

   andptr(tmp, count-1);

   jccb(Assembler::notZero, brLabel);

 }

 // Perform abort ratio calculation, set no_rtm bit if high ratio

 // input:  rtm_counters_Reg (RTMLockingCounters* address)

 // tmpReg, rtm_counters_Reg and flags are killed

 void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg,

                                                  Register rtm_counters_Reg,

                                                  RTMLockingCounters* rtm_counters,

                                                  Metadata* method_data) {

   Label L_done, L_check_always_rtm1, L_check_always_rtm2;

   if (RTMLockingCalculationDelay > 0) {

     // Delay calculation

     movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg);

     testptr(tmpReg, tmpReg);

     jccb(Assembler::equal, L_done);

   }

   // Abort ratio calculation only if abort_count > RTMAbortThreshold

   //   Aborted transactions = abort_count * 100

   //   All transactions = total_count *  RTMTotalCountIncrRate

   //   Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)

   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset()));

   cmpptr(tmpReg, RTMAbortThreshold);

   jccb(Assembler::below, L_check_always_rtm2);

   imulptr(tmpReg, tmpReg, 100);

   Register scrReg = rtm_counters_Reg;

   movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));

   imulptr(scrReg, scrReg, RTMTotalCountIncrRate);

   imulptr(scrReg, scrReg, RTMAbortRatio);

   cmpptr(tmpReg, scrReg);

   jccb(Assembler::below, L_check_always_rtm1);

   if (method_data != NULL) {

     // set rtm_state to "no rtm" in MDO

     mov_metadata(tmpReg, method_data);

     if (os::is_MP()) {

       lock();

     }

     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM);

   }

   jmpb(L_done);

   bind(L_check_always_rtm1);

   // Reload RTMLockingCounters* address

   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));

   bind(L_check_always_rtm2);

   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));

   cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate);

   jccb(Assembler::below, L_done);

   if (method_data != NULL) {

     // set rtm_state to "always rtm" in MDO

     mov_metadata(tmpReg, method_data);

     if (os::is_MP()) {

       lock();

     }

     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM);

   }

   bind(L_done);

 }

 // Update counters and perform abort ratio calculation

 // input:  abort_status_Reg

 // rtm_counters_Reg, flags are killed

 void MacroAssembler::rtm_profiling(Register abort_status_Reg,

                                    Register rtm_counters_Reg,

                                    RTMLockingCounters* rtm_counters,

                                    Metadata* method_data,

                                    bool profile_rtm) {

   assert(rtm_counters != NULL, "should not be NULL when profiling RTM");

   // update rtm counters based on rax value at abort

   // reads abort_status_Reg, updates flags

   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));

   rtm_counters_update(abort_status_Reg, rtm_counters_Reg);

   if (profile_rtm) {

     // Save abort status because abort_status_Reg is used by following code.

     if (RTMRetryCount > 0) {

       push(abort_status_Reg);

     }

     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");

     rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data);

     // restore abort status

     if (RTMRetryCount > 0) {

       pop(abort_status_Reg);

     }

   }

 }

 // Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4)

 // inputs: retry_count_Reg

 //       : abort_status_Reg

 // output: retry_count_Reg decremented by 1

 // flags are killed

 void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) {

   Label doneRetry;

   assert(abort_status_Reg == rax, "");

   // The abort reason bits are in eax (see all states in rtmLocking.hpp)

   // 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4)

   // if reason is in 0x6 and retry count != 0 then retry

   andptr(abort_status_Reg, 0x6);

   jccb(Assembler::zero, doneRetry);

   testl(retry_count_Reg, retry_count_Reg);

   jccb(Assembler::zero, doneRetry);

   pause();

   decrementl(retry_count_Reg);

   jmp(retryLabel);

   bind(doneRetry);

 }

 // Spin and retry if lock is busy,

 // inputs: box_Reg (monitor address)

 //       : retry_count_Reg

 // output: retry_count_Reg decremented by 1

 //       : clear z flag if retry count exceeded

 // tmp_Reg, scr_Reg, flags are killed

 void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg,

                                             Register tmp_Reg, Register scr_Reg, Label& retryLabel) {

   Label SpinLoop, SpinExit, doneRetry;

   // Clean monitor_value bit to get valid pointer

   int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value;

   testl(retry_count_Reg, retry_count_Reg);

   jccb(Assembler::zero, doneRetry);

   decrementl(retry_count_Reg);

   movptr(scr_Reg, RTMSpinLoopCount);

   bind(SpinLoop);

   pause();

   decrementl(scr_Reg);

   jccb(Assembler::lessEqual, SpinExit);

   movptr(tmp_Reg, Address(box_Reg, owner_offset));

   testptr(tmp_Reg, tmp_Reg);

   jccb(Assembler::notZero, SpinLoop);

   bind(SpinExit);

   jmp(retryLabel);

   bind(doneRetry);

   incrementl(retry_count_Reg); // clear z flag

 }

 // Use RTM for normal stack locks

 // Input: objReg (object to lock)

 void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg,

                                        Register retry_on_abort_count_Reg,

                                        RTMLockingCounters* stack_rtm_counters,

                                        Metadata* method_data, bool profile_rtm,

                                        Label& DONE_LABEL, Label& IsInflated) {

   assert(UseRTMForStackLocks, "why call this otherwise?");

   assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");

   assert(tmpReg == rax, "");

   assert(scrReg == rdx, "");

   Label L_rtm_retry, L_decrement_retry, L_on_abort;

   if (RTMRetryCount > 0) {

     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort

     bind(L_rtm_retry);

   }

   movptr(tmpReg, Address(objReg, 0));

   testptr(tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased

   jcc(Assembler::notZero, IsInflated);

   if (PrintPreciseRTMLockingStatistics || profile_rtm) {

     Label L_noincrement;

     if (RTMTotalCountIncrRate > 1) {

       // tmpReg, scrReg and flags are killed

       branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement);

     }

     assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");

     atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg);

     bind(L_noincrement);

   }

   xbegin(L_on_abort);

   movptr(tmpReg, Address(objReg, 0));       // fetch markword

   andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits

   cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked

   jcc(Assembler::equal, DONE_LABEL);        // all done if unlocked

   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX

   if (UseRTMXendForLockBusy) {

     xend();

     movptr(abort_status_Reg, 0x2);   // Set the abort status to 2 (so we can retry)

     jmp(L_decrement_retry);

   }

   else {

     xabort(0);

   }

   bind(L_on_abort);

   if (PrintPreciseRTMLockingStatistics || profile_rtm) {

     rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm);

   }

   bind(L_decrement_retry);

   if (RTMRetryCount > 0) {

     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)

     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);

   }

 }

 // Use RTM for inflating locks

 // inputs: objReg (object to lock)

 //         boxReg (on-stack box address (displaced header location) - KILLED)

 //         tmpReg (ObjectMonitor address + 2(monitor_value))

 void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,

                                           Register scrReg, Register retry_on_busy_count_Reg,

                                           Register retry_on_abort_count_Reg,

                                           RTMLockingCounters* rtm_counters,

                                           Metadata* method_data, bool profile_rtm,

                                           Label& DONE_LABEL) {

   assert(UseRTMLocking, "why call this otherwise?");

   assert(tmpReg == rax, "");

   assert(scrReg == rdx, "");

   Label L_rtm_retry, L_decrement_retry, L_on_abort;

   // Clean monitor_value bit to get valid pointer

   int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value;

   // Without cast to int32_t a movptr will destroy r10 which is typically obj

   movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));

   movptr(boxReg, tmpReg); // Save ObjectMonitor address

   if (RTMRetryCount > 0) {

     movl(retry_on_busy_count_Reg, RTMRetryCount);  // Retry on lock busy

     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort

     bind(L_rtm_retry);

   }

   if (PrintPreciseRTMLockingStatistics || profile_rtm) {

     Label L_noincrement;

     if (RTMTotalCountIncrRate > 1) {

       // tmpReg, scrReg and flags are killed

       branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement);

     }

     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");

     atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);

     bind(L_noincrement);

   }

   xbegin(L_on_abort);

   movptr(tmpReg, Address(objReg, 0));

   movptr(tmpReg, Address(tmpReg, owner_offset));

   testptr(tmpReg, tmpReg);

   jcc(Assembler::zero, DONE_LABEL);

   if (UseRTMXendForLockBusy) {

     xend();

     jmp(L_decrement_retry);

   }

   else {

     xabort(0);

   }

   bind(L_on_abort);

   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX

   if (PrintPreciseRTMLockingStatistics || profile_rtm) {

     rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm);

   }

   if (RTMRetryCount > 0) {

     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)

     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);

   }

   movptr(tmpReg, Address(boxReg, owner_offset)) ;

   testptr(tmpReg, tmpReg) ;

   jccb(Assembler::notZero, L_decrement_retry) ;

   // Appears unlocked - try to swing _owner from null to non-null.

   // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.

 #ifdef _LP64

   Register threadReg = r15_thread;

 #else

   get_thread(scrReg);

   Register threadReg = scrReg;

 #endif

   if (os::is_MP()) {

     lock();

   }

   cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg

   if (RTMRetryCount > 0) {

     // success done else retry

     jccb(Assembler::equal, DONE_LABEL) ;

     bind(L_decrement_retry);

     // Spin and retry if lock is busy.

     rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);

   }

   else {

     bind(L_decrement_retry);

   }

 }

 #endif //  INCLUDE_RTM_OPT

 // Fast_Lock and Fast_Unlock used by C2

 // Because the transitions from emitted code to the runtime

 // monitorenter/exit helper stubs are so slow it's critical that

 // we inline both the stack-locking fast-path and the inflated fast path.

 //

 // See also: cmpFastLock and cmpFastUnlock.

 //

 // What follows is a specialized inline transliteration of the code

 // in slow_enter() and slow_exit().  If we're concerned about I$ bloat

 // another option would be to emit TrySlowEnter and TrySlowExit methods

 // at startup-time.  These methods would accept arguments as

 // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure

 // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply

 // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.

 // In practice, however, the # of lock sites is bounded and is usually small.

 // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer

 // if the processor uses simple bimodal branch predictors keyed by EIP

 // Since the helper routines would be called from multiple synchronization

 // sites.

 //

 // An even better approach would be write "MonitorEnter()" and "MonitorExit()"

 // in java - using j.u.c and unsafe - and just bind the lock and unlock sites

 // to those specialized methods.  That'd give us a mostly platform-independent

 // implementation that the JITs could optimize and inline at their pleasure.

 // Done correctly, the only time we'd need to cross to native could would be

 // to park() or unpark() threads.  We'd also need a few more unsafe operators

 // to (a) prevent compiler-JIT reordering of non-volatile accesses, and

 // (b) explicit barriers or fence operations.

 //

 // TODO:

 //

 // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).

 //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.

 //    Given TLAB allocation, Self is usually manifested in a register, so passing it into

 //    the lock operators would typically be faster than reifying Self.

 //

 // *  Ideally I'd define the primitives as:

 //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.

 //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED

 //    Unfortunately ADLC bugs prevent us from expressing the ideal form.

 //    Instead, we're stuck with a rather awkward and brittle register assignments below.

 //    Furthermore the register assignments are overconstrained, possibly resulting in

 //    sub-optimal code near the synchronization site.

 //

 // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.

 //    Alternately, use a better sp-proximity test.

 //

 // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.

 //    Either one is sufficient to uniquely identify a thread.

 //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.

 //

 // *  Intrinsify notify() and notifyAll() for the common cases where the

 //    object is locked by the calling thread but the waitlist is empty.

 //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().

 //

 // *  use jccb and jmpb instead of jcc and jmp to improve code density.

 //    But beware of excessive branch density on AMD Opterons.

 //

 // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success

 //    or failure of the fast-path.  If the fast-path fails then we pass

 //    control to the slow-path, typically in C.  In Fast_Lock and

 //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2

 //    will emit a conditional branch immediately after the node.

 //    So we have branches to branches and lots of ICC.ZF games.

 //    Instead, it might be better to have C2 pass a "FailureLabel"

 //    into Fast_Lock and Fast_Unlock.  In the case of success, control

 //    will drop through the node.  ICC.ZF is undefined at exit.

 //    In the case of failure, the node will branch directly to the

 //    FailureLabel

 // obj: object to lock

 // box: on-stack box address (displaced header location) - KILLED

 // rax,: tmp -- KILLED

 // scr: tmp -- KILLED

 void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,

                                Register scrReg, Register cx1Reg, Register cx2Reg,

                                BiasedLockingCounters* counters,

                                RTMLockingCounters* rtm_counters,

                                RTMLockingCounters* stack_rtm_counters,

                                Metadata* method_data,

                                bool use_rtm, bool profile_rtm) {

   // Ensure the register assignents are disjoint

   assert(tmpReg == rax, "");

   if (use_rtm) {

     assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg);

   } else {

     assert(cx1Reg == noreg, "");

     assert(cx2Reg == noreg, "");

     assert_different_registers(objReg, boxReg, tmpReg, scrReg);

   }

   if (counters != NULL) {

     atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);

   }

   if (EmitSync & 1) {

       // set box->dhw = unused_mark (3)

       // Force all sync thru slow-path: slow_enter() and slow_exit()

       movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));

       cmpptr (rsp, (int32_t)NULL_WORD);

   } else

   if (EmitSync & 2) {

       Label DONE_LABEL ;

       if (UseBiasedLocking) {

          // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.

          biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);

       }

       movptr(tmpReg, Address(objReg, 0));           // fetch markword

       orptr (tmpReg, 0x1);

       movptr(Address(boxReg, 0), tmpReg);           // Anticipate successful CAS

       if (os::is_MP()) {

         lock();

       }

       cmpxchgptr(boxReg, Address(objReg, 0));       // Updates tmpReg

       jccb(Assembler::equal, DONE_LABEL);

       // Recursive locking

       subptr(tmpReg, rsp);

       andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );

       movptr(Address(boxReg, 0), tmpReg);

       bind(DONE_LABEL);

   } else {

     // Possible cases that we'll encounter in fast_lock

     // ------------------------------------------------

     // * Inflated

     //    -- unlocked

     //    -- Locked

     //       = by self

     //       = by other

     // * biased

     //    -- by Self

     //    -- by other

     // * neutral

     // * stack-locked

     //    -- by self

     //       = sp-proximity test hits

     //       = sp-proximity test generates false-negative

     //    -- by other

     //

     Label IsInflated, DONE_LABEL;

     // it's stack-locked, biased or neutral

     // TODO: optimize away redundant LDs of obj->mark and improve the markword triage

     // order to reduce the number of conditional branches in the most common cases.

     // Beware -- there's a subtle invariant that fetch of the markword

     // at [FETCH], below, will never observe a biased encoding (*101b).

     // If this invariant is not held we risk exclusion (safety) failure.

     if (UseBiasedLocking && !UseOptoBiasInlining) {

       biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, counters);

     }

 #if INCLUDE_RTM_OPT

     if (UseRTMForStackLocks && use_rtm) {

       rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,

                         stack_rtm_counters, method_data, profile_rtm,

                         DONE_LABEL, IsInflated);

     }

 #endif // INCLUDE_RTM_OPT

     movptr(tmpReg, Address(objReg, 0));          // [FETCH]

     testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased

     jccb(Assembler::notZero, IsInflated);

     // Attempt stack-locking ...

     orptr (tmpReg, markOopDesc::unlocked_value);

     movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS

     if (os::is_MP()) {

       lock();

     }

     cmpxchgptr(boxReg, Address(objReg, 0));      // Updates tmpReg

     if (counters != NULL) {

       cond_inc32(Assembler::equal,

                  ExternalAddress((address)counters->fast_path_entry_count_addr()));

     }

     jcc(Assembler::equal, DONE_LABEL);           // Success

     // Recursive locking.

     // The object is stack-locked: markword contains stack pointer to BasicLock.

     // Locked by current thread if difference with current SP is less than one page.

     subptr(tmpReg, rsp);

     // Next instruction set ZFlag == 1 (Success) if difference is less then one page.

     andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );

     movptr(Address(boxReg, 0), tmpReg);

     if (counters != NULL) {

       cond_inc32(Assembler::equal,

                  ExternalAddress((address)counters->fast_path_entry_count_addr()));

     }

     jmp(DONE_LABEL);

     bind(IsInflated);

     // The object is inflated. tmpReg contains pointer to ObjectMonitor* + 2(monitor_value)

 #if INCLUDE_RTM_OPT

     // Use the same RTM locking code in 32- and 64-bit VM.

     if (use_rtm) {

       rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg,

                            rtm_counters, method_data, profile_rtm, DONE_LABEL);

     } else {

 #endif // INCLUDE_RTM_OPT

 #ifndef _LP64

     // The object is inflated.

     //

     // TODO-FIXME: eliminate the ugly use of manifest constants:

     //   Use markOopDesc::monitor_value instead of "2".

     //   use markOop::unused_mark() instead of "3".

     // The tmpReg value is an objectMonitor reference ORed with

     // markOopDesc::monitor_value (2).   We can either convert tmpReg to an

     // objectmonitor pointer by masking off the "2" bit or we can just

     // use tmpReg as an objectmonitor pointer but bias the objectmonitor

     // field offsets with "-2" to compensate for and annul the low-order tag bit.

     //

     // I use the latter as it avoids AGI stalls.

     // As such, we write "mov r, [tmpReg+OFFSETOF(Owner)-2]"

     // instead of "mov r, [tmpReg+OFFSETOF(Owner)]".

     //

     #define OFFSET_SKEWED(f) ((ObjectMonitor::f ## _offset_in_bytes())-2)

     // boxReg refers to the on-stack BasicLock in the current frame.

     // We'd like to write:

     //   set box->_displaced_header = markOop::unused_mark().  Any non-0 value suffices.

     // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers

     // additional latency as we have another ST in the store buffer that must drain.

     if (EmitSync & 8192) {

        movptr(Address(boxReg, 0), 3);            // results in ST-before-CAS penalty

        get_thread (scrReg);

        movptr(boxReg, tmpReg);                    // consider: LEA box, [tmp-2]

        movptr(tmpReg, NULL_WORD);                 // consider: xor vs mov

        if (os::is_MP()) {

          lock();

        }

        cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));

     } else

     if ((EmitSync & 128) == 0) {                      // avoid ST-before-CAS

        movptr(scrReg, boxReg);

        movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2]

        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes

        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {

           // prefetchw [eax + Offset(_owner)-2]

           prefetchw(Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));

        }

        if ((EmitSync & 64) == 0) {

          // Optimistic form: consider XORL tmpReg,tmpReg

          movptr(tmpReg, NULL_WORD);

        } else {

          // Can suffer RTS->RTO upgrades on shared or cold $ lines

          // Test-And-CAS instead of CAS

          movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));   // rax, = m->_owner

          testptr(tmpReg, tmpReg);                   // Locked ?

          jccb  (Assembler::notZero, DONE_LABEL);

        }

        // Appears unlocked - try to swing _owner from null to non-null.

        // Ideally, I'd manifest "Self" with get_thread and then attempt

        // to CAS the register containing Self into m->Owner.

        // But we don't have enough registers, so instead we can either try to CAS

        // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds

        // we later store "Self" into m->Owner.  Transiently storing a stack address

        // (rsp or the address of the box) into  m->owner is harmless.

        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.

        if (os::is_MP()) {

          lock();

        }

        cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));

        movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3

        jccb  (Assembler::notZero, DONE_LABEL);

        get_thread (scrReg);                    // beware: clobbers ICCs

        movptr(Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2), scrReg);

        xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success

        // If the CAS fails we can either retry or pass control to the slow-path.

        // We use the latter tactic.

        // Pass the CAS result in the icc.ZFlag into DONE_LABEL

        // If the CAS was successful ...

        //   Self has acquired the lock

        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.

        // Intentional fall-through into DONE_LABEL ...

     } else {

        movptr(Address(boxReg, 0), intptr_t(markOopDesc::unused_mark()));  // results in ST-before-CAS penalty

        movptr(boxReg, tmpReg);

        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes

        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {

           // prefetchw [eax + Offset(_owner)-2]

           prefetchw(Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));

        }

        if ((EmitSync & 64) == 0) {

          // Optimistic form

          xorptr  (tmpReg, tmpReg);

        } else {

          // Can suffer RTS->RTO upgrades on shared or cold $ lines

          movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));   // rax, = m->_owner

          testptr(tmpReg, tmpReg);                   // Locked ?

          jccb  (Assembler::notZero, DONE_LABEL);

        }

        // Appears unlocked - try to swing _owner from null to non-null.

        // Use either "Self" (in scr) or rsp as thread identity in _owner.

        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.

        get_thread (scrReg);

        if (os::is_MP()) {

          lock();

        }

        cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));

        // If the CAS fails we can either retry or pass control to the slow-path.

        // We use the latter tactic.

        // Pass the CAS result in the icc.ZFlag into DONE_LABEL

        // If the CAS was successful ...

        //   Self has acquired the lock

        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.

        // Intentional fall-through into DONE_LABEL ...

     }

 #else // _LP64

     // It's inflated

     // TODO: someday avoid the ST-before-CAS penalty by

     // relocating (deferring) the following ST.

     // We should also think about trying a CAS without having

     // fetched _owner.  If the CAS is successful we may

     // avoid an RTO->RTS upgrade on the $line.

     // Without cast to int32_t a movptr will destroy r10 which is typically obj

     movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));

     movptr (boxReg, tmpReg);

     movptr (tmpReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));

     testptr(tmpReg, tmpReg);

     jccb   (Assembler::notZero, DONE_LABEL);

     // It's inflated and appears unlocked

     if (os::is_MP()) {

       lock();

     }

     cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));

     // Intentional fall-through into DONE_LABEL ...

 #endif // _LP64

 #if INCLUDE_RTM_OPT

     } // use_rtm()

 #endif

     // DONE_LABEL is a hot target - we'd really like to place it at the

     // start of cache line by padding with NOPs.

     // See the AMD and Intel software optimization manuals for the

     // most efficient "long" NOP encodings.

     // Unfortunately none of our alignment mechanisms suffice.

     bind(DONE_LABEL);

     // At DONE_LABEL the icc ZFlag is set as follows ...

     // Fast_Unlock uses the same protocol.

     // ZFlag == 1 -> Success

     // ZFlag == 0 -> Failure - force control through the slow-path

   }

 }

 // obj: object to unlock

 // box: box address (displaced header location), killed.  Must be EAX.

 // tmp: killed, cannot be obj nor box.

 //

 // Some commentary on balanced locking:

 //

 // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.

 // Methods that don't have provably balanced locking are forced to run in the

 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.

 // The interpreter provides two properties:

 // I1:  At return-time the interpreter automatically and quietly unlocks any

 //      objects acquired the current activation (frame).  Recall that the

 //      interpreter maintains an on-stack list of locks currently held by

 //      a frame.

 // I2:  If a method attempts to unlock an object that is not held by the

 //      the frame the interpreter throws IMSX.

 //

 // Lets say A(), which has provably balanced locking, acquires O and then calls B().

 // B() doesn't have provably balanced locking so it runs in the interpreter.

 // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O

 // is still locked by A().

 //

 // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:

 // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter

 // should not be unlocked by "normal" java-level locking and vice-versa.  The specification

 // doesn't specify what will occur if a program engages in such mixed-mode locking, however.

 void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {

   assert(boxReg == rax, "");

   assert_different_registers(objReg, boxReg, tmpReg);

   if (EmitSync & 4) {

     // Disable - inhibit all inlining.  Force control through the slow-path

     cmpptr (rsp, 0);

   } else

   if (EmitSync & 8) {

     Label DONE_LABEL;

     if (UseBiasedLocking) {

        biased_locking_exit(objReg, tmpReg, DONE_LABEL);

     }

     // Classic stack-locking code ...

     // Check whether the displaced header is 0

     //(=> recursive unlock)

     movptr(tmpReg, Address(boxReg, 0));

     testptr(tmpReg, tmpReg);

     jccb(Assembler::zero, DONE_LABEL);

     // If not recursive lock, reset the header to displaced header

     if (os::is_MP()) {

       lock();

     }

     cmpxchgptr(tmpReg, Address(objReg, 0));   // Uses RAX which is box

     bind(DONE_LABEL);

   } else {

     Label DONE_LABEL, Stacked, CheckSucc;

     // Critically, the biased locking test must have precedence over

     // and appear before the (box->dhw == 0) recursive stack-lock test.

     if (UseBiasedLocking && !UseOptoBiasInlining) {

        biased_locking_exit(objReg, tmpReg, DONE_LABEL);

     }

 #if INCLUDE_RTM_OPT

     if (UseRTMForStackLocks && use_rtm) {

       assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");

       Label L_regular_unlock;

       movptr(tmpReg, Address(objReg, 0));           // fetch markword

       andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits

       cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked

       jccb(Assembler::notEqual, L_regular_unlock);  // if !HLE RegularLock

       xend();                                       // otherwise end...

       jmp(DONE_LABEL);                              // ... and we're done

       bind(L_regular_unlock);

     }

 #endif

     cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header

     jcc   (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock

     movptr(tmpReg, Address(objReg, 0));             // Examine the object's markword

     testptr(tmpReg, markOopDesc::monitor_value);    // Inflated?

     jccb  (Assembler::zero, Stacked);

     // It's inflated.

 #if INCLUDE_RTM_OPT

     if (use_rtm) {

       Label L_regular_inflated_unlock;

       // Clean monitor_value bit to get valid pointer

       int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value;

       movptr(boxReg, Address(tmpReg, owner_offset));

       testptr(boxReg, boxReg);

       jccb(Assembler::notZero, L_regular_inflated_unlock);

       xend();

       jmpb(DONE_LABEL);

       bind(L_regular_inflated_unlock);

     }

 #endif

     // Despite our balanced locking property we still check that m->_owner == Self

     // as java routines or native JNI code called by this thread might

     // have released the lock.

     // Refer to the comments in synchronizer.cpp for how we might encode extra

     // state in _succ so we can avoid fetching EntryList|cxq.

     //

     // I'd like to add more cases in fast_lock() and fast_unlock() --

     // such as recursive enter and exit -- but we have to be wary of

     // I$ bloat, T$ effects and BP$ effects.

     //

     // If there's no contention try a 1-0 exit.  That is, exit without

     // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how

     // we detect and recover from the race that the 1-0 exit admits.

     //

     // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier

     // before it STs null into _owner, releasing the lock.  Updates

     // to data protected by the critical section must be visible before

     // we drop the lock (and thus before any other thread could acquire

     // the lock and observe the fields protected by the lock).

     // IA32's memory-model is SPO, so STs are ordered with respect to

     // each other and there's no need for an explicit barrier (fence).

     // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.

 #ifndef _LP64

     get_thread (boxReg);

     if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {

       // prefetchw [ebx + Offset(_owner)-2]

       prefetchw(Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));

     }

     // Note that we could employ various encoding schemes to reduce

     // the number of loads below (currently 4) to just 2 or 3.

     // Refer to the comments in synchronizer.cpp.

     // In practice the chain of fetches doesn't seem to impact performance, however.

     if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {

        // Attempt to reduce branch density - AMD's branch predictor.

        xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));

        orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2));

        orptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2));

        orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2));

        jccb  (Assembler::notZero, DONE_LABEL);

        movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD);

        jmpb  (DONE_LABEL);

     } else {

        xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));

        orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2));

        jccb  (Assembler::notZero, DONE_LABEL);

        movptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2));

        orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2));

        jccb  (Assembler::notZero, CheckSucc);

        movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD);

        jmpb  (DONE_LABEL);

     }

     // The Following code fragment (EmitSync & 65536) improves the performance of

     // contended applications and contended synchronization microbenchmarks.

     // Unfortunately the emission of the code - even though not executed - causes regressions

     // in scimark and jetstream, evidently because of $ effects.  Replacing the code

     // with an equal number of never-executed NOPs results in the same regression.

     // We leave it off by default.

     if ((EmitSync & 65536) != 0) {

        Label LSuccess, LGoSlowPath ;

        bind  (CheckSucc);

        // Optional pre-test ... it's safe to elide this

        if ((EmitSync & 16) == 0) {

           cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD);

           jccb  (Assembler::zero, LGoSlowPath);

        }

        // We have a classic Dekker-style idiom:

        //    ST m->_owner = 0 ; MEMBAR; LD m->_succ

        // There are a number of ways to implement the barrier:

        // (1) lock:andl &m->_owner, 0

        //     is fast, but mask doesn't currently support the "ANDL M,IMM32" form.

        //     LOCK: ANDL [ebx+Offset(_Owner)-2], 0

        //     Encodes as 81 31 OFF32 IMM32 or 83 63 OFF8 IMM8

        // (2) If supported, an explicit MFENCE is appealing.

        //     In older IA32 processors MFENCE is slower than lock:add or xchg

        //     particularly if the write-buffer is full as might be the case if

        //     if stores closely precede the fence or fence-equivalent instruction.

        //     In more modern implementations MFENCE appears faster, however.

        // (3) In lieu of an explicit fence, use lock:addl to the top-of-stack

        //     The $lines underlying the top-of-stack should be in M-state.

        //     The locked add instruction is serializing, of course.

        // (4) Use xchg, which is serializing

        //     mov boxReg, 0; xchgl boxReg, [tmpReg + Offset(_owner)-2] also works

        // (5) ST m->_owner = 0 and then execute lock:orl &m->_succ, 0.

        //     The integer condition codes will tell us if succ was 0.

        //     Since _succ and _owner should reside in the same $line and

        //     we just stored into _owner, it's likely that the $line

        //     remains in M-state for the lock:orl.

        //

        // We currently use (3), although it's likely that switching to (2)

        // is correct for the future.

        movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD);

        if (os::is_MP()) {

           if (VM_Version::supports_sse2() && 1 == FenceInstruction) {

             mfence();

           } else {

             lock (); addptr(Address(rsp, 0), 0);

           }

        }

        // Ratify _succ remains non-null

        cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0);

        jccb  (Assembler::notZero, LSuccess);

        xorptr(boxReg, boxReg);                  // box is really EAX

        if (os::is_MP()) { lock(); }

        cmpxchgptr(rsp, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));

        jccb  (Assembler::notEqual, LSuccess);

        // Since we're low on registers we installed rsp as a placeholding in _owner.

        // Now install Self over rsp.  This is safe as we're transitioning from

        // non-null to non=null

        get_thread (boxReg);

        movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), boxReg);

        // Intentional fall-through into LGoSlowPath ...

        bind  (LGoSlowPath);

        orptr(boxReg, 1);                      // set ICC.ZF=0 to indicate failure

        jmpb  (DONE_LABEL);

        bind  (LSuccess);

        xorptr(boxReg, boxReg);                 // set ICC.ZF=1 to indicate success

        jmpb  (DONE_LABEL);

     }

     bind (Stacked);

     // It's not inflated and it's not recursively stack-locked and it's not biased.

     // It must be stack-locked.

     // Try to reset the header to displaced header.

     // The "box" value on the stack is stable, so we can reload

     // and be assured we observe the same value as above.

     movptr(tmpReg, Address(boxReg, 0));

     if (os::is_MP()) {

       lock();

     }

     cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box

     // Intention fall-thru into DONE_LABEL

     // DONE_LABEL is a hot target - we'd really like to place it at the

     // start of cache line by padding with NOPs.

     // See the AMD and Intel software optimization manuals for the

     // most efficient "long" NOP encodings.

     // Unfortunately none of our alignment mechanisms suffice.

     if ((EmitSync & 65536) == 0) {

        bind (CheckSucc);

     }

 #else // _LP64

     // It's inflated

     movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));

     xorptr(boxReg, r15_thread);

     orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2));

     jccb  (Assembler::notZero, DONE_LABEL);

     movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2));

     orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2));

     jccb  (Assembler::notZero, CheckSucc);

     movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD);

     jmpb  (DONE_LABEL);

     if ((EmitSync & 65536) == 0) {

       Label LSuccess, LGoSlowPath ;

       bind  (CheckSucc);

       cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD);

       jccb  (Assembler::zero, LGoSlowPath);

       // I'd much rather use lock:andl m->_owner, 0 as it's faster than the

       // the explicit ST;MEMBAR combination, but masm doesn't currently support

       // "ANDQ M,IMM".  Don't use MFENCE here.  lock:add to TOS, xchg, etc

       // are all faster when the write buffer is populated.

       movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD);

       if (os::is_MP()) {

          lock (); addl (Address(rsp, 0), 0);

       }

       cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD);

       jccb  (Assembler::notZero, LSuccess);

       movptr (boxReg, (int32_t)NULL_WORD);                   // box is really EAX

       if (os::is_MP()) { lock(); }

       cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));

       jccb  (Assembler::notEqual, LSuccess);

       // Intentional fall-through into slow-path

       bind  (LGoSlowPath);

       orl   (boxReg, 1);                      // set ICC.ZF=0 to indicate failure

       jmpb  (DONE_LABEL);

       bind  (LSuccess);

       testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success

       jmpb  (DONE_LABEL);

     }

     bind  (Stacked);

     movptr(tmpReg, Address (boxReg, 0));      // re-fetch

     if (os::is_MP()) { lock(); }

     cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box

     if (EmitSync & 65536) {

        bind (CheckSucc);

     }

 #endif

     bind(DONE_LABEL);

     // Avoid branch to branch on AMD processors

     if (EmitSync & 32768) {

        nop();

     }

   }

 }

 #endif // COMPILER2

 void MacroAssembler::c2bool(Register x) {

   // implements x == 0 ? 0 : 1

   // note: must only look at least-significant byte of x

   //       since C-style booleans are stored in one byte

   //       only! (was bug)

   andl(x, 0xFF);

   setb(Assembler::notZero, x);

 }

 // Wouldn't need if AddressLiteral version had new name

 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {

   Assembler::call(L, rtype);

 }

 void MacroAssembler::call(Register entry) {

   Assembler::call(entry);

 }

 void MacroAssembler::call(AddressLiteral entry) {

   if (reachable(entry)) {

     Assembler::call_literal(entry.target(), entry.rspec());

   } else {

     lea(rscratch1, entry);

     Assembler::call(rscratch1);

   }

 }

 void MacroAssembler::ic_call(address entry) {

   RelocationHolder rh = virtual_call_Relocation::spec(pc());

   movptr(rax, (intptr_t)Universe::non_oop_word());

   call(AddressLiteral(entry, rh));

 }

 // Implementation of call_VM versions

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              bool check_exceptions) {

   Label C, E;

   call(C, relocInfo::none);

   jmp(E);

   bind(C);

   call_VM_helper(oop_result, entry_point, 0, check_exceptions);

   ret(0);

   bind(E);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              Register arg_1,

                              bool check_exceptions) {

   Label C, E;

   call(C, relocInfo::none);

   jmp(E);

   bind(C);

   pass_arg1(this, arg_1);

   call_VM_helper(oop_result, entry_point, 1, check_exceptions);

   ret(0);

   bind(E);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              bool check_exceptions) {

   Label C, E;

   call(C, relocInfo::none);

   jmp(E);

   bind(C);

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   call_VM_helper(oop_result, entry_point, 2, check_exceptions);

   ret(0);

   bind(E);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              Register arg_3,

                              bool check_exceptions) {

   Label C, E;

   call(C, relocInfo::none);

   jmp(E);

   bind(C);

   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));

   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));

   pass_arg3(this, arg_3);

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   call_VM_helper(oop_result, entry_point, 3, check_exceptions);

   ret(0);

   bind(E);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              int number_of_arguments,

                              bool check_exceptions) {

   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);

   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              Register arg_1,

                              bool check_exceptions) {

   pass_arg1(this, arg_1);

   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              bool check_exceptions) {

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              Register arg_3,

                              bool check_exceptions) {

   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));

   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));

   pass_arg3(this, arg_3);

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);

 }

 void MacroAssembler::super_call_VM(Register oop_result,

                                    Register last_java_sp,

                                    address entry_point,

                                    int number_of_arguments,

                                    bool check_exceptions) {

   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);

   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);

 }

 void MacroAssembler::super_call_VM(Register oop_result,

                                    Register last_java_sp,

                                    address entry_point,

                                    Register arg_1,

                                    bool check_exceptions) {

   pass_arg1(this, arg_1);

   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);

 }

 void MacroAssembler::super_call_VM(Register oop_result,

                                    Register last_java_sp,

                                    address entry_point,

                                    Register arg_1,

                                    Register arg_2,

                                    bool check_exceptions) {

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);

 }

 void MacroAssembler::super_call_VM(Register oop_result,

                                    Register last_java_sp,

                                    address entry_point,

                                    Register arg_1,

                                    Register arg_2,

                                    Register arg_3,

                                    bool check_exceptions) {

   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));

   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));

   pass_arg3(this, arg_3);

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   pass_arg1(this, arg_1);

   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);

 }

 void MacroAssembler::call_VM_base(Register oop_result,

                                   Register java_thread,

                                   Register last_java_sp,

                                   address  entry_point,

                                   int      number_of_arguments,

                                   bool     check_exceptions) {

   // determine java_thread register

   if (!java_thread->is_valid()) {

 #ifdef _LP64

     java_thread = r15_thread;

 #else

     java_thread = rdi;

     get_thread(java_thread);

 #endif // LP64

   }

   // determine last_java_sp register

   if (!last_java_sp->is_valid()) {

     last_java_sp = rsp;

   }

   // debugging support

   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");

   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));

 #ifdef ASSERT

   // TraceBytecodes does not use r12 but saves it over the call, so don't verify

   // r12 is the heapbase.

   LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)

 #endif // ASSERT

   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");

   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");

   // push java thread (becomes first argument of C function)

   NOT_LP64(push(java_thread); number_of_arguments++);

   LP64_ONLY(mov(c_rarg0, r15_thread));

   // set last Java frame before call

   assert(last_java_sp != rbp, "can't use ebp/rbp");

   // Only interpreter should have to set fp

   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);

   // do the call, remove parameters

   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);

   // restore the thread (cannot use the pushed argument since arguments

   // may be overwritten by C code generated by an optimizing compiler);

   // however can use the register value directly if it is callee saved.

   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {

     // rdi & rsi (also r15) are callee saved -> nothing to do

 #ifdef ASSERT

     guarantee(java_thread != rax, "change this code");

     push(rax);

     { Label L;

       get_thread(rax);

       cmpptr(java_thread, rax);

       jcc(Assembler::equal, L);

       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");

       bind(L);

     }

     pop(rax);

 #endif

   } else {

     get_thread(java_thread);

   }

   // reset last Java frame

   // Only interpreter should have to clear fp

   reset_last_Java_frame(java_thread, true, false);

 #ifndef CC_INTERP

    // C++ interp handles this in the interpreter

   check_and_handle_popframe(java_thread);

   check_and_handle_earlyret(java_thread);

 #endif /* CC_INTERP */

   if (check_exceptions) {

     // check for pending exceptions (java_thread is set upon return)

     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);

 #ifndef _LP64

     jump_cc(Assembler::notEqual,

             RuntimeAddress(StubRoutines::forward_exception_entry()));

 #else

     // This used to conditionally jump to forward_exception however it is

     // possible if we relocate that the branch will not reach. So we must jump

     // around so we can always reach

     Label ok;

     jcc(Assembler::equal, ok);

     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));

     bind(ok);

 #endif // LP64

   }

   // get oop result if there is one and reset the value in the thread

   if (oop_result->is_valid()) {

     get_vm_result(oop_result, java_thread);

   }

 }

 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {

   // Calculate the value for last_Java_sp

   // somewhat subtle. call_VM does an intermediate call

   // which places a return address on the stack just under the

   // stack pointer as the user finsihed with it. This allows

   // use to retrieve last_Java_pc from last_Java_sp[-1].

   // On 32bit we then have to push additional args on the stack to accomplish

   // the actual requested call. On 64bit call_VM only can use register args

   // so the only extra space is the return address that call_VM created.

   // This hopefully explains the calculations here.

 #ifdef _LP64

   // We've pushed one address, correct last_Java_sp

   lea(rax, Address(rsp, wordSize));

 #else

   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));

 #endif // LP64

   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {

   call_VM_leaf_base(entry_point, number_of_arguments);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {

   pass_arg0(this, arg_0);

   call_VM_leaf(entry_point, 1);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {

   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));

   pass_arg1(this, arg_1);

   pass_arg0(this, arg_0);

   call_VM_leaf(entry_point, 2);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {

   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));

   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));

   pass_arg2(this, arg_2);

   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));

   pass_arg1(this, arg_1);

   pass_arg0(this, arg_0);

   call_VM_leaf(entry_point, 3);

 }