Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.

  * Copyright (c) 2017, 2020, Loongson Technology. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "precompiled.hpp"

 #include "asm/assembler.hpp"

 #include "asm/assembler.inline.hpp"

 #include "asm/macroAssembler.inline.hpp"

 #include "compiler/disassembler.hpp"

 #include "gc_interface/collectedHeap.inline.hpp"

 #include "interpreter/interpreter.hpp"

 #include "memory/cardTableModRefBS.hpp"

 #include "memory/resourceArea.hpp"

 #include "memory/universe.hpp"

 #include "prims/methodHandles.hpp"

 #include "runtime/biasedLocking.hpp"

 #include "runtime/interfaceSupport.hpp"

 #include "runtime/objectMonitor.hpp"

 #include "runtime/os.hpp"

 #include "runtime/sharedRuntime.hpp"

 #include "runtime/stubRoutines.hpp"

 #include "utilities/macros.hpp"

 #if INCLUDE_ALL_GCS

 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"

 #include "gc_implementation/g1/heapRegion.hpp"

 #endif // INCLUDE_ALL_GCS

 // Implementation of MacroAssembler

 intptr_t MacroAssembler::i[32] = {0};

 float MacroAssembler::f[32] = {0.0};

 void MacroAssembler::print(outputStream *s) {

   unsigned int k;

   for(k=0; k<sizeof(i)/sizeof(i[0]); k++) {

     s->print_cr("i%d = 0x%.16lx", k, i[k]);

   }

   s->cr();

   for(k=0; k<sizeof(f)/sizeof(f[0]); k++) {

     s->print_cr("f%d = %f", k, f[k]);

   }

   s->cr();

 }

 int MacroAssembler::i_offset(unsigned int k) { return (intptr_t)&((MacroAssembler*)0)->i[k]; }

 int MacroAssembler::f_offset(unsigned int k) { return (intptr_t)&((MacroAssembler*)0)->f[k]; }

 void MacroAssembler::save_registers(MacroAssembler *masm) {

 #define __ masm->

   for(int k=0; k<32; k++) {

     __ sw (as_Register(k), A0, i_offset(k));

   }

   for(int k=0; k<32; k++) {

     __ swc1 (as_FloatRegister(k), A0, f_offset(k));

   }

 #undef __

 }

 void MacroAssembler::restore_registers(MacroAssembler *masm) {

 #define __ masm->

   for(int k=0; k<32; k++) {

     __ lw (as_Register(k), A0, i_offset(k));

   }

   for(int k=0; k<32; k++) {

     __ lwc1 (as_FloatRegister(k), A0, f_offset(k));

   }

 #undef __

 }

 void MacroAssembler::pd_patch_instruction(address branch, address target) {

   jint& stub_inst = *(jint*) branch;

   jint *pc = (jint *)branch;

   if((opcode(stub_inst) == special_op) && (special(stub_inst) == dadd_op)) {

     //b_far:

     //  move(AT, RA); // dadd

     //  emit_long(insn_ORRI(regimm_op, 0, bgezal_op, 1));

     //  nop();

     //  lui(T9, 0); // to be patched

     //  ori(T9, 0);

     //  daddu(T9, T9, RA);

     //  move(RA, AT);

     //  jr(T9);

     assert(opcode(pc[3]) == lui_op

         && opcode(pc[4]) == ori_op

         && special(pc[5]) == daddu_op, "Not a branch label patch");

     if(!(opcode(pc[3]) == lui_op

           && opcode(pc[4]) == ori_op

           && special(pc[5]) == daddu_op)) { tty->print_cr("Not a branch label patch"); }

     int offset = target - branch;

     if (!is_simm16(offset)) {

       pc[3] = (pc[3] & 0xffff0000) | high16(offset - 12);

       pc[4] = (pc[4] & 0xffff0000) | low16(offset - 12);

     } else {

       // revert to "beq + nop"

       CodeBuffer cb(branch, 4 * 10);

       MacroAssembler masm(&cb);

 #define __ masm.

       __ b(target);

       __ delayed()->nop();

       __ nop();

       __ nop();

       __ nop();

       __ nop();

       __ nop();

       __ nop();

     }

     return;

   } else if (special(pc[4]) == jr_op

              && opcode(pc[4]) == special_op

              && (((opcode(pc[0]) == lui_op) || opcode(pc[0]) == daddiu_op) || (opcode(pc[0]) == ori_op))) {

     //jmp_far:

     //  patchable_set48(T9, target);

     //  jr(T9);

     //  nop();

     CodeBuffer cb(branch, 4 * 4);

     MacroAssembler masm(&cb);

     masm.patchable_set48(T9, (long)(target));

     return;

   }

 #ifndef PRODUCT

   if (!is_simm16((target - branch - 4) >> 2)) {

     tty->print_cr("Illegal patching: branch = 0x%lx, target = 0x%lx", branch, target);

     tty->print_cr("======= Start decoding at branch = 0x%lx =======", branch);

     Disassembler::decode(branch - 4 * 16, branch + 4 * 16, tty);

     tty->print_cr("======= End of decoding =======");

   }

 #endif

   stub_inst = patched_branch(target - branch, stub_inst, 0);

 }

 static inline address first_cache_address() {

   return CodeCache::low_bound() + sizeof(HeapBlock::Header);

 }

 static inline address last_cache_address() {

   return CodeCache::high_bound() - Assembler::InstructionSize;

 }

 int MacroAssembler::call_size(address target, bool far, bool patchable) {

   if (patchable) return 6 << Assembler::LogInstructionSize;

   if (!far) return 2 << Assembler::LogInstructionSize; // jal + nop

   return (insts_for_set64((jlong)target) + 2) << Assembler::LogInstructionSize;

 }

 // Can we reach target using jal/j from anywhere

 // in the code cache (because code can be relocated)?

 bool MacroAssembler::reachable_from_cache(address target) {

   address cl = first_cache_address();

   address ch = last_cache_address();

   return (cl <= target) && (target <= ch) && fit_in_jal(cl, ch);

 }

 void MacroAssembler::general_jump(address target) {

   if (reachable_from_cache(target)) {

     j(target);

     delayed()->nop();

   } else {

     set64(T9, (long)target);

     jr(T9);

     delayed()->nop();

   }

 }

 int MacroAssembler::insts_for_general_jump(address target) {

   if (reachable_from_cache(target)) {

     //j(target);

     //nop();

     return 2;

   } else {

     //set64(T9, (long)target);

     //jr(T9);

     //nop();

     return insts_for_set64((jlong)target) + 2;

   }

 }

 void MacroAssembler::patchable_jump(address target) {

   if (reachable_from_cache(target)) {

     nop();

     nop();

     nop();

     nop();

     j(target);

     delayed()->nop();

   } else {

     patchable_set48(T9, (long)target);

     jr(T9);

     delayed()->nop();

   }

 }

 int MacroAssembler::insts_for_patchable_jump(address target) {

   return 6;

 }

 void MacroAssembler::general_call(address target) {

   if (reachable_from_cache(target)) {

     jal(target);

     delayed()->nop();

   } else {

     set64(T9, (long)target);

     jalr(T9);

     delayed()->nop();

   }

 }

 int MacroAssembler::insts_for_general_call(address target) {

   if (reachable_from_cache(target)) {

     //jal(target);

     //nop();

     return 2;

   } else {

     //set64(T9, (long)target);

     //jalr(T9);

     //nop();

     return insts_for_set64((jlong)target) + 2;

   }

 }

 void MacroAssembler::patchable_call(address target) {

   if (reachable_from_cache(target)) {

     nop();

     nop();

     nop();

     nop();

     jal(target);

     delayed()->nop();

   } else {

     patchable_set48(T9, (long)target);

     jalr(T9);

     delayed()->nop();

   }

 }

 int MacroAssembler::insts_for_patchable_call(address target) {

   return 6;

 }

 void MacroAssembler::beq_far(Register rs, Register rt, address entry) {

   u_char * cur_pc = pc();

   // Near/Far jump

   if(is_simm16((entry - pc() - 4) / 4)) {

     Assembler::beq(rs, rt, offset(entry));

   } else {

     Label not_jump;

     bne(rs, rt, not_jump);

     delayed()->nop();

     b_far(entry);

     delayed()->nop();

     bind(not_jump);

     has_delay_slot();

   }

 }

 void MacroAssembler::beq_far(Register rs, Register rt, Label& L) {

   if (L.is_bound()) {

     beq_far(rs, rt, target(L));

   } else {

     u_char * cur_pc = pc();

     Label not_jump;

     bne(rs, rt, not_jump);

     delayed()->nop();

     b_far(L);

     delayed()->nop();

     bind(not_jump);

     has_delay_slot();

   }

 }

 void MacroAssembler::bne_far(Register rs, Register rt, address entry) {

   u_char * cur_pc = pc();

   //Near/Far jump

   if(is_simm16((entry - pc() - 4) / 4)) {

     Assembler::bne(rs, rt, offset(entry));

   } else {

     Label not_jump;

     beq(rs, rt, not_jump);

     delayed()->nop();

     b_far(entry);

     delayed()->nop();

     bind(not_jump);

     has_delay_slot();

   }

 }

 void MacroAssembler::bne_far(Register rs, Register rt, Label& L) {

   if (L.is_bound()) {

     bne_far(rs, rt, target(L));

   } else {

     u_char * cur_pc = pc();

     Label not_jump;

     beq(rs, rt, not_jump);

     delayed()->nop();

     b_far(L);

     delayed()->nop();

     bind(not_jump);

     has_delay_slot();

   }

 }

 void MacroAssembler::beq_long(Register rs, Register rt, Label& L) {

   Label not_taken;

   bne(rs, rt, not_taken);

   delayed()->nop();

   jmp_far(L);

   bind(not_taken);

 }

 void MacroAssembler::bne_long(Register rs, Register rt, Label& L) {

   Label not_taken;

   beq(rs, rt, not_taken);

   delayed()->nop();

   jmp_far(L);

   bind(not_taken);

 }

 void MacroAssembler::bc1t_long(Label& L) {

   Label not_taken;

   bc1f(not_taken);

   delayed()->nop();

   jmp_far(L);

   bind(not_taken);

 }

 void MacroAssembler::bc1f_long(Label& L) {

   Label not_taken;

   bc1t(not_taken);

   delayed()->nop();

   jmp_far(L);

   bind(not_taken);

 }

 void MacroAssembler::b_far(Label& L) {

   if (L.is_bound()) {

     b_far(target(L));

   } else {

     volatile address dest = target(L);

 //

 // MacroAssembler::pd_patch_instruction branch=55651ed514, target=55651ef6d8

 //   0x00000055651ed514: dadd at, ra, zero

 //   0x00000055651ed518: [4110001]bgezal zero, 0x00000055651ed520

 //

 //   0x00000055651ed51c: sll zero, zero, 0

 //   0x00000055651ed520: lui t9, 0x0

 //   0x00000055651ed524: ori t9, t9, 0x21b8

 //   0x00000055651ed528: daddu t9, t9, ra

 //   0x00000055651ed52c: dadd ra, at, zero

 //   0x00000055651ed530: jr t9

 //   0x00000055651ed534: sll zero, zero, 0

 //

     move(AT, RA);

     emit_long(insn_ORRI(regimm_op, 0, bgezal_op, 1));

     nop();

     lui(T9, 0); // to be patched

     ori(T9, T9, 0);

     daddu(T9, T9, RA);

     move(RA, AT);

     jr(T9);

   }

 }

 void MacroAssembler::b_far(address entry) {

   u_char * cur_pc = pc();

   // Near/Far jump

   if(is_simm16((entry - pc() - 4) / 4)) {

     b(offset(entry));

   } else {

     // address must be bounded

     move(AT, RA);

     emit_long(insn_ORRI(regimm_op, 0, bgezal_op, 1));

     nop();

     li32(T9, entry - pc());

     daddu(T9, T9, RA);

     move(RA, AT);

     jr(T9);

   }

 }

 void MacroAssembler::ld_ptr(Register rt, Register offset, Register base) {

   addu_long(AT, base, offset);

   ld_ptr(rt, 0, AT);

 }

 void MacroAssembler::st_ptr(Register rt, Register offset, Register base) {

   addu_long(AT, base, offset);

   st_ptr(rt, 0, AT);

 }

 void MacroAssembler::ld_long(Register rt, Register offset, Register base) {

   addu_long(AT, base, offset);

   ld_long(rt, 0, AT);

 }

 void MacroAssembler::st_long(Register rt, Register offset, Register base) {

   addu_long(AT, base, offset);

   st_long(rt, 0, AT);

 }

 Address MacroAssembler::as_Address(AddressLiteral adr) {

   return Address(adr.target(), adr.rspec());

 }

 Address MacroAssembler::as_Address(ArrayAddress adr) {

   return Address::make_array(adr);

 }

 // tmp_reg1 and tmp_reg2 should be saved outside of atomic_inc32 (caller saved).

 void MacroAssembler::atomic_inc32(address counter_addr, int inc, Register tmp_reg1, Register tmp_reg2) {

   Label again;

   li(tmp_reg1, counter_addr);

   bind(again);

   if (UseSyncLevel >= 10000 || UseSyncLevel == 1000 || UseSyncLevel == 4000) sync();

   ll(tmp_reg2, tmp_reg1, 0);

   addi(tmp_reg2, tmp_reg2, inc);

   sc(tmp_reg2, tmp_reg1, 0);

   beq(tmp_reg2, R0, again);

   delayed()->nop();

 }

 int MacroAssembler::biased_locking_enter(Register lock_reg,

                                          Register obj_reg,

                                          Register swap_reg,

                                          Register tmp_reg,

                                          bool swap_reg_contains_mark,

                                          Label& done,

                                          Label* slow_case,

                                          BiasedLockingCounters* counters) {

   assert(UseBiasedLocking, "why call this otherwise?");

   bool need_tmp_reg = false;

   if (tmp_reg == noreg) {

     need_tmp_reg = true;

     tmp_reg = T9;

   }

   assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg, AT);

   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");

   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());

   Address saved_mark_addr(lock_reg, 0);

   // Biased locking

   // See whether the lock is currently biased toward our thread and

   // whether the epoch is still valid

   // Note that the runtime guarantees sufficient alignment of JavaThread

   // pointers to allow age to be placed into low bits

   // First check to see whether biasing is even enabled for this object

   Label cas_label;

   int null_check_offset = -1;

   if (!swap_reg_contains_mark) {

     null_check_offset = offset();

     ld_ptr(swap_reg, mark_addr);

   }

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   move(tmp_reg, swap_reg);

   andi(tmp_reg, tmp_reg, markOopDesc::biased_lock_mask_in_place);

 #ifdef _LP64

   daddi(AT, R0, markOopDesc::biased_lock_pattern);

   dsub(AT, AT, tmp_reg);

 #else

   addi(AT, R0, markOopDesc::biased_lock_pattern);

   sub(AT, AT, tmp_reg);

 #endif

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   bne(AT, R0, cas_label);

   delayed()->nop();

   // The bias pattern is present in the object's header. Need to check

   // whether the bias owner and the epoch are both still current.

   // Note that because there is no current thread register on MIPS we

   // need to store off the mark word we read out of the object to

   // avoid reloading it and needing to recheck invariants below. This

   // store is unfortunate but it makes the overall code shorter and

   // simpler.

   st_ptr(swap_reg, saved_mark_addr);

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   if (swap_reg_contains_mark) {

     null_check_offset = offset();

   }

   load_prototype_header(tmp_reg, obj_reg);

   xorr(tmp_reg, tmp_reg, swap_reg);

   get_thread(swap_reg);

   xorr(swap_reg, swap_reg, tmp_reg);

   move(AT, ~((int) markOopDesc::age_mask_in_place));

   andr(swap_reg, swap_reg, AT);

   if (PrintBiasedLockingStatistics) {

     Label L;

     bne(swap_reg, R0, L);

     delayed()->nop();

     push(tmp_reg);

     push(A0);

     atomic_inc32((address)BiasedLocking::biased_lock_entry_count_addr(), 1, A0, tmp_reg);

     pop(A0);

     pop(tmp_reg);

     bind(L);

   }

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   beq(swap_reg, R0, done);

   delayed()->nop();

   Label try_revoke_bias;

   Label try_rebias;

   // At this point we know that the header has the bias pattern and

   // that we are not the bias owner in the current epoch. We need to

   // figure out more details about the state of the header in order to

   // know what operations can be legally performed on the object's

   // header.

   // If the low three bits in the xor result aren't clear, that means

   // the prototype header is no longer biased and we have to revoke

   // the bias on this object.

   move(AT, markOopDesc::biased_lock_mask_in_place);

   andr(AT, swap_reg, AT);

   bne(AT, R0, try_revoke_bias);

   delayed()->nop();

   // Biasing is still enabled for this data type. See whether the

   // epoch of the current bias is still valid, meaning that the epoch

   // bits of the mark word are equal to the epoch bits of the

   // prototype header. (Note that the prototype header's epoch bits

   // only change at a safepoint.) If not, attempt to rebias the object

   // toward the current thread. Note that we must be absolutely sure

   // that the current epoch is invalid in order to do this because

   // otherwise the manipulations it performs on the mark word are

   // illegal.

   move(AT, markOopDesc::epoch_mask_in_place);

   andr(AT,swap_reg, AT);

   bne(AT, R0, try_rebias);

   delayed()->nop();

   // The epoch of the current bias is still valid but we know nothing

   // about the owner; it might be set or it might be clear. Try to

   // acquire the bias of the object using an atomic operation. If this

   // fails we will go in to the runtime to revoke the object's bias.

   // Note that we first construct the presumed unbiased header so we

   // don't accidentally blow away another thread's valid bias.

   ld_ptr(swap_reg, saved_mark_addr);

   move(AT, markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);

   andr(swap_reg, swap_reg, AT);

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   get_thread(tmp_reg);

   orr(tmp_reg, tmp_reg, swap_reg);

   //if (os::is_MP()) {

   //  sync();

   //}

   cmpxchg(tmp_reg, Address(obj_reg, 0), swap_reg);

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   // If the biasing toward our thread failed, this means that

   // another thread succeeded in biasing it toward itself and we

   // need to revoke that bias. The revocation will occur in the

   // interpreter runtime in the slow case.

   if (PrintBiasedLockingStatistics) {

     Label L;

     bne(AT, R0, L);

     delayed()->nop();

     push(tmp_reg);

     push(A0);

     atomic_inc32((address)BiasedLocking::anonymously_biased_lock_entry_count_addr(), 1, A0, tmp_reg);

     pop(A0);

     pop(tmp_reg);

     bind(L);

   }

   if (slow_case != NULL) {

     beq_far(AT, R0, *slow_case);

     delayed()->nop();

   }

   b(done);

   delayed()->nop();

   bind(try_rebias);

   // At this point we know the epoch has expired, meaning that the

   // current "bias owner", if any, is actually invalid. Under these

   // circumstances _only_, we are allowed to use the current header's

   // value as the comparison value when doing the cas to acquire the

   // bias in the current epoch. In other words, we allow transfer of

   // the bias from one thread to another directly in this situation.

   //

   // FIXME: due to a lack of registers we currently blow away the age

   // bits in this situation. Should attempt to preserve them.

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   load_prototype_header(tmp_reg, obj_reg);

   get_thread(swap_reg);

   orr(tmp_reg, tmp_reg, swap_reg);

   ld_ptr(swap_reg, saved_mark_addr);

   //if (os::is_MP()) {

   //  sync();

   //}

   cmpxchg(tmp_reg, Address(obj_reg, 0), swap_reg);

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   // If the biasing toward our thread failed, then another thread

   // succeeded in biasing it toward itself and we need to revoke that

   // bias. The revocation will occur in the runtime in the slow case.

   if (PrintBiasedLockingStatistics) {

     Label L;

     bne(AT, R0, L);

     delayed()->nop();

     push(AT);

     push(tmp_reg);

     atomic_inc32((address)BiasedLocking::rebiased_lock_entry_count_addr(), 1, AT, tmp_reg);

     pop(tmp_reg);

     pop(AT);

     bind(L);

   }

   if (slow_case != NULL) {

     beq_far(AT, R0, *slow_case);

     delayed()->nop();

   }

   b(done);

   delayed()->nop();

   bind(try_revoke_bias);

   // The prototype mark in the klass doesn't have the bias bit set any

   // more, indicating that objects of this data type are not supposed

   // to be biased any more. We are going to try to reset the mark of

   // this object to the prototype value and fall through to the

   // CAS-based locking scheme. Note that if our CAS fails, it means

   // that another thread raced us for the privilege of revoking the

   // bias of this particular object, so it's okay to continue in the

   // normal locking code.

   //

   // FIXME: due to a lack of registers we currently blow away the age

   // bits in this situation. Should attempt to preserve them.

   ld_ptr(swap_reg, saved_mark_addr);

   if (need_tmp_reg) {

     push(tmp_reg);

   }

   load_prototype_header(tmp_reg, obj_reg);

   //if (os::is_MP()) {

   // lock();

   //}

   cmpxchg(tmp_reg, Address(obj_reg, 0), swap_reg);

   if (need_tmp_reg) {

     pop(tmp_reg);

   }

   // Fall through to the normal CAS-based lock, because no matter what

   // the result of the above CAS, some thread must have succeeded in

   // removing the bias bit from the object's header.

   if (PrintBiasedLockingStatistics) {

     Label L;

     bne(AT, R0, L);

     delayed()->nop();

     push(AT);

     push(tmp_reg);

     atomic_inc32((address)BiasedLocking::revoked_lock_entry_count_addr(), 1, AT, tmp_reg);

     pop(tmp_reg);

     pop(AT);

     bind(L);

   }

   bind(cas_label);

   return null_check_offset;

 }

 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {

   assert(UseBiasedLocking, "why call this otherwise?");

   // Check for biased locking unlock case, which is a no-op

   // Note: we do not have to check the thread ID for two reasons.

   // First, the interpreter checks for IllegalMonitorStateException at

   // a higher level. Second, if the bias was revoked while we held the

   // lock, the object could not be rebiased toward another thread, so

   // the bias bit would be clear.

 #ifdef _LP64

   ld(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));

   andi(temp_reg, temp_reg, markOopDesc::biased_lock_mask_in_place);

   daddi(AT, R0, markOopDesc::biased_lock_pattern);

 #else

   lw(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));

   andi(temp_reg, temp_reg, markOopDesc::biased_lock_mask_in_place);

   addi(AT, R0, markOopDesc::biased_lock_pattern);

 #endif

   beq(AT, temp_reg, done);

   delayed()->nop();

 }

 // the stack pointer adjustment is needed. see InterpreterMacroAssembler::super_call_VM_leaf

 // this method will handle the stack problem, you need not to preserve the stack space for the argument now

 void MacroAssembler::call_VM_leaf_base(address entry_point, int number_of_arguments) {

   Label L, E;

   assert(number_of_arguments <= 4, "just check");

   andi(AT, SP, 0xf);

   beq(AT, R0, L);

   delayed()->nop();

   daddi(SP, SP, -8);

   call(entry_point, relocInfo::runtime_call_type);

   delayed()->nop();

   daddi(SP, SP, 8);

   b(E);

   delayed()->nop();

   bind(L);

   call(entry_point, relocInfo::runtime_call_type);

   delayed()->nop();

   bind(E);

 }

 void MacroAssembler::jmp(address entry) {

   patchable_set48(T9, (long)entry);

   jr(T9);

 }

 void MacroAssembler::jmp(address entry, relocInfo::relocType rtype) {

   switch (rtype) {

     case relocInfo::runtime_call_type:

     case relocInfo::none:

       jmp(entry);

       break;

     default:

       {

       InstructionMark im(this);

       relocate(rtype);

       patchable_set48(T9, (long)entry);

       jr(T9);

       }

       break;

   }

 }

 void MacroAssembler::jmp_far(Label& L) {

   if (L.is_bound()) {

     address entry = target(L);

     assert(entry != NULL, "jmp most probably wrong");

     InstructionMark im(this);

     relocate(relocInfo::internal_word_type);

     patchable_set48(T9, (long)entry);

   } else {

     InstructionMark im(this);

     L.add_patch_at(code(), locator());

     relocate(relocInfo::internal_word_type);

     patchable_set48(T9, (long)pc());

   }

   jr(T9);

   delayed()->nop();

 }

 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {

   int oop_index;

   if (obj) {

     oop_index = oop_recorder()->find_index(obj);

   } else {

     oop_index = oop_recorder()->allocate_metadata_index(obj);

   }

   relocate(metadata_Relocation::spec(oop_index));

   patchable_set48(AT, (long)obj);

   sd(AT, dst);

 }

 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {

   int oop_index;

   if (obj) {

     oop_index = oop_recorder()->find_index(obj);

   } else {

     oop_index = oop_recorder()->allocate_metadata_index(obj);

   }

   relocate(metadata_Relocation::spec(oop_index));

   patchable_set48(dst, (long)obj);

 }

 void MacroAssembler::call(address entry) {

 // c/c++ code assume T9 is entry point, so we just always move entry to t9

 // maybe there is some more graceful method to handle this. FIXME

 // For more info, see class NativeCall.

 #ifndef _LP64

   move(T9, (int)entry);

 #else

   patchable_set48(T9, (long)entry);

 #endif

   jalr(T9);

 }

 void MacroAssembler::call(address entry, relocInfo::relocType rtype) {

   switch (rtype) {

     case relocInfo::runtime_call_type:

     case relocInfo::none:

       call(entry);

       break;

     default:

       {

   InstructionMark im(this);

   relocate(rtype);

   call(entry);

       }

       break;

   }

 }

 void MacroAssembler::call(address entry, RelocationHolder& rh)

 {

   switch (rh.type()) {

     case relocInfo::runtime_call_type:

     case relocInfo::none:

       call(entry);

       break;

     default:

       {

   InstructionMark im(this);

   relocate(rh);

   call(entry);

       }

       break;

   }

 }

 void MacroAssembler::ic_call(address entry) {

   RelocationHolder rh = virtual_call_Relocation::spec(pc());

   patchable_set48(IC_Klass, (long)Universe::non_oop_word());

   assert(entry != NULL, "call most probably wrong");

   InstructionMark im(this);

   relocate(rh);

   patchable_call(entry);

 }

 void MacroAssembler::c2bool(Register r) {

   Label L;

   Assembler::beq(r, R0, L);

   delayed()->nop();

   move(r, 1);

   bind(L);

 }

 #ifndef PRODUCT

 extern "C" void findpc(intptr_t x);

 #endif

 void MacroAssembler::debug(char* msg/*, RegistersForDebugging* regs*/) {

   if ( ShowMessageBoxOnError ) {

     JavaThreadState saved_state = JavaThread::current()->thread_state();

     JavaThread::current()->set_thread_state(_thread_in_vm);

     {

       // In order to get locks work, we need to fake a in_VM state

       ttyLocker ttyl;

       ::tty->print_cr("EXECUTION STOPPED: %s\n", msg);

       if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {

   BytecodeCounter::print();

       }

     }

     ThreadStateTransition::transition(JavaThread::current(), _thread_in_vm, saved_state);

   }

   else

     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);

 }

 void MacroAssembler::stop(const char* msg) {

   li(A0, (long)msg);

 #ifndef _LP64

   //reserver space for argument.

   addiu(SP, SP, - 1 * wordSize);

 #endif

   call(CAST_FROM_FN_PTR(address, MacroAssembler::debug), relocInfo::runtime_call_type);

   delayed()->nop();

 #ifndef _LP64

   //restore space for argument

   addiu(SP, SP, 1 * wordSize);

 #endif

   brk(17);

 }

 void MacroAssembler::warn(const char* msg) {

 #ifdef _LP64

   pushad();

   li(A0, (long)msg);

   push(S2);

   move(AT, -(StackAlignmentInBytes));

   move(S2, SP);     // use S2 as a sender SP holder

   andr(SP, SP, AT); // align stack as required by ABI

   call(CAST_FROM_FN_PTR(address, MacroAssembler::debug), relocInfo::runtime_call_type);

   delayed()->nop();

   move(SP, S2);     // use S2 as a sender SP holder

   pop(S2);

   popad();

 #else

   pushad();

   addi(SP, SP, -4);

   sw(A0, SP, -1 * wordSize);

   li(A0, (long)msg);

   addi(SP, SP, -1 * wordSize);

   call(CAST_FROM_FN_PTR(address, MacroAssembler::debug), relocInfo::runtime_call_type);

   delayed()->nop();

   addi(SP, SP, 1 * wordSize);

   lw(A0, SP, -1 * wordSize);

   addi(SP, SP, 4);

   popad();

 #endif

 }

 void MacroAssembler::print_reg(Register reg) {

   void * cur_pc = pc();

   pushad();

   NOT_LP64(push(FP);)

   li(A0, (long)reg->name());

   if (reg == SP)

     addiu(A1, SP, wordSize * 23); //23 registers saved in pushad()

   else if (reg == A0)

     ld(A1, SP, wordSize * 19); //A0 has been modified by li(A0, (long)reg->name()). Ugly Code!

   else

     move(A1, reg);

   li(A2, (long)cur_pc);

   push(S2);

   move(AT, -(StackAlignmentInBytes));

   move(S2, SP);     // use S2 as a sender SP holder

   andr(SP, SP, AT); // align stack as required by ABI

   call(CAST_FROM_FN_PTR(address, SharedRuntime::print_reg_with_pc),relocInfo::runtime_call_type);

   delayed()->nop();

   move(SP, S2);     // use S2 as a sender SP holder

   pop(S2);

   NOT_LP64(pop(FP);)

   popad();

 }

 void MacroAssembler::print_reg(FloatRegister reg) {

   void * cur_pc = pc();

   pushad();

   NOT_LP64(push(FP);)

   li(A0, (long)reg->name());

   push(S2);

   move(AT, -(StackAlignmentInBytes));

   move(S2, SP);     // use S2 as a sender SP holder

   andr(SP, SP, AT); // align stack as required by ABI

   call(CAST_FROM_FN_PTR(address, SharedRuntime::print_str),relocInfo::runtime_call_type);

   delayed()->nop();

   move(SP, S2);     // use S2 as a sender SP holder

   pop(S2);

   NOT_LP64(pop(FP);)

   popad();

   pushad();

   NOT_LP64(push(FP);)

   move(FP, SP);

   move(AT, -(StackAlignmentInBytes));

   andr(SP , SP , AT);

   mov_d(F12, reg);

   call(CAST_FROM_FN_PTR(address, SharedRuntime::print_double),relocInfo::runtime_call_type);

   delayed()->nop();

   move(SP, FP);

   NOT_LP64(pop(FP);)

   popad();

 }

 void MacroAssembler::increment(Register reg, int imm) {

   if (!imm) return;

   if (is_simm16(imm)) {

 #ifdef _LP64

     daddiu(reg, reg, imm);

 #else

     addiu(reg, reg, imm);

 #endif

   } else {

     move(AT, imm);

 #ifdef _LP64

     daddu(reg, reg, AT);

 #else

     addu(reg, reg, AT);

 #endif

   }

 }

 void MacroAssembler::decrement(Register reg, int imm) {

   increment(reg, -imm);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              bool check_exceptions) {

   call_VM_helper(oop_result, entry_point, 0, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              Register arg_1,

                              bool check_exceptions) {

   if (arg_1!=A1) move(A1, arg_1);

   call_VM_helper(oop_result, entry_point, 1, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              bool check_exceptions) {

   if (arg_1!=A1) move(A1, arg_1);

   if (arg_2!=A2) move(A2, arg_2);

   assert(arg_2 != A1, "smashed argument");

   call_VM_helper(oop_result, entry_point, 2, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              Register arg_3,

                              bool check_exceptions) {

   if (arg_1!=A1) move(A1, arg_1);

   if (arg_2!=A2) move(A2, arg_2); assert(arg_2 != A1, "smashed argument");

   if (arg_3!=A3) move(A3, arg_3); assert(arg_3 != A1 && arg_3 != A2, "smashed argument");

   call_VM_helper(oop_result, entry_point, 3, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              int number_of_arguments,

                              bool check_exceptions) {

   call_VM_base(oop_result, NOREG, last_java_sp, entry_point, number_of_arguments, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              Register arg_1,

                              bool check_exceptions) {

   if (arg_1 != A1) move(A1, arg_1);

   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              bool check_exceptions) {

   if (arg_1 != A1) move(A1, arg_1);

   if (arg_2 != A2) move(A2, arg_2); assert(arg_2 != A1, "smashed argument");

   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);

 }

 void MacroAssembler::call_VM(Register oop_result,

                              Register last_java_sp,

                              address entry_point,

                              Register arg_1,

                              Register arg_2,

                              Register arg_3,

                              bool check_exceptions) {

   if (arg_1 != A1) move(A1, arg_1);

   if (arg_2 != A2) move(A2, arg_2); assert(arg_2 != A1, "smashed argument");

   if (arg_3 != A3) move(A3, arg_3); assert(arg_3 != A1 && arg_3 != A2, "smashed argument");

   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);

 }

 void MacroAssembler::call_VM_base(Register oop_result,

                                   Register java_thread,

                                   Register last_java_sp,

                                   address  entry_point,

                                   int      number_of_arguments,

                                   bool     check_exceptions) {

   address before_call_pc;

   // determine java_thread register

   if (!java_thread->is_valid()) {

 #ifndef OPT_THREAD

     java_thread = T2;

     get_thread(java_thread);

 #else

     java_thread = TREG;

 #endif

   }

   // determine last_java_sp register

   if (!last_java_sp->is_valid()) {

     last_java_sp = SP;

   }

   // debugging support

   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");

   assert(number_of_arguments <= 4   , "cannot have negative number of arguments");

   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");

   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");

   assert(last_java_sp != FP, "this code doesn't work for last_java_sp == fp, which currently can't portably work anyway since C2 doesn't save fp");

   // set last Java frame before call

   before_call_pc = (address)pc();

   set_last_Java_frame(java_thread, last_java_sp, FP, before_call_pc);

   // do the call

   move(A0, java_thread);

   call(entry_point, relocInfo::runtime_call_type);

   delayed()->nop();

   // restore the thread (cannot use the pushed argument since arguments

   // may be overwritten by C code generated by an optimizing compiler);

   // however can use the register value directly if it is callee saved.

 #ifndef OPT_THREAD

   get_thread(java_thread);

 #else

 #ifdef ASSERT

   {

     Label L;

     get_thread(AT);

     beq(java_thread, AT, L);

     delayed()->nop();

     stop("MacroAssembler::call_VM_base: TREG not callee saved?");

     bind(L);

   }

 #endif

 #endif

   // discard thread and arguments

   ld_ptr(SP, java_thread, in_bytes(JavaThread::last_Java_sp_offset()));

   // reset last Java frame

   reset_last_Java_frame(java_thread, false);

   check_and_handle_popframe(java_thread);

   check_and_handle_earlyret(java_thread);

   if (check_exceptions) {

     // check for pending exceptions (java_thread is set upon return)

     Label L;

 #ifdef _LP64

     ld(AT, java_thread, in_bytes(Thread::pending_exception_offset()));

 #else

     lw(AT, java_thread, in_bytes(Thread::pending_exception_offset()));

 #endif

     beq(AT, R0, L);

     delayed()->nop();

     li(AT, before_call_pc);

     push(AT);

     jmp(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);

     delayed()->nop();

     bind(L);

   }

   // get oop result if there is one and reset the value in the thread

   if (oop_result->is_valid()) {

 #ifdef _LP64

     ld(oop_result, java_thread, in_bytes(JavaThread::vm_result_offset()));

     sd(R0, java_thread, in_bytes(JavaThread::vm_result_offset()));

 #else

     lw(oop_result, java_thread, in_bytes(JavaThread::vm_result_offset()));

     sw(R0, java_thread, in_bytes(JavaThread::vm_result_offset()));

 #endif

     verify_oop(oop_result);

   }

 }

 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {

   move(V0, SP);

   //we also reserve space for java_thread here

 #ifndef _LP64

   daddi(SP, SP, (1 + number_of_arguments) * (- wordSize));

 #endif

   move(AT, -(StackAlignmentInBytes));

   andr(SP, SP, AT);

   call_VM_base(oop_result, NOREG, V0, entry_point, number_of_arguments, check_exceptions);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {

   call_VM_leaf_base(entry_point, number_of_arguments);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {

   if (arg_0 != A0) move(A0, arg_0);

   call_VM_leaf(entry_point, 1);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {

   if (arg_0 != A0) move(A0, arg_0);

   if (arg_1 != A1) move(A1, arg_1); assert(arg_1 != A0, "smashed argument");

   call_VM_leaf(entry_point, 2);

 }

 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {

   if (arg_0 != A0) move(A0, arg_0);

   if (arg_1 != A1) move(A1, arg_1); assert(arg_1 != A0, "smashed argument");

   if (arg_2 != A2) move(A2, arg_2); assert(arg_2 != A0 && arg_2 != A1, "smashed argument");

   call_VM_leaf(entry_point, 3);

 }

 void MacroAssembler::super_call_VM_leaf(address entry_point) {

   MacroAssembler::call_VM_leaf_base(entry_point, 0);

 }

 void MacroAssembler::super_call_VM_leaf(address entry_point,

                                                    Register arg_1) {

   if (arg_1 != A0) move(A0, arg_1);

   MacroAssembler::call_VM_leaf_base(entry_point, 1);

 }

 void MacroAssembler::super_call_VM_leaf(address entry_point,

                                                    Register arg_1,

                                                    Register arg_2) {

   if (arg_1 != A0) move(A0, arg_1);

   if (arg_2 != A1) move(A1, arg_2); assert(arg_2 != A0, "smashed argument");

   MacroAssembler::call_VM_leaf_base(entry_point, 2);

 }

 void MacroAssembler::super_call_VM_leaf(address entry_point,

                                                    Register arg_1,

                                                    Register arg_2,

                                                    Register arg_3) {

   if (arg_1 != A0) move(A0, arg_1);

   if (arg_2 != A1) move(A1, arg_2); assert(arg_2 != A0, "smashed argument");

   if (arg_3 != A2) move(A2, arg_3); assert(arg_3 != A0 && arg_3 != A1, "smashed argument");

   MacroAssembler::call_VM_leaf_base(entry_point, 3);

 }

 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {

 }

 void MacroAssembler::check_and_handle_popframe(Register java_thread) {

 }

 void MacroAssembler::null_check(Register reg, int offset) {

   if (needs_explicit_null_check(offset)) {

     // provoke OS NULL exception if reg = NULL by

     // accessing M[reg] w/o changing any (non-CC) registers

     // NOTE: cmpl is plenty here to provoke a segv

     lw(AT, reg, 0);

   } else {

     // nothing to do, (later) access of M[reg + offset]

     // will provoke OS NULL exception if reg = NULL

   }

 }

 void MacroAssembler::enter() {

   push2(RA, FP);

   move(FP, SP);

 }

 void MacroAssembler::leave() {

 #ifndef _LP64

   addi(SP, FP, 2 * wordSize);

   lw(RA, SP, - 1 * wordSize);

   lw(FP, SP, - 2 * wordSize);

 #else

   daddi(SP, FP, 2 * wordSize);

   ld(RA, SP, - 1 * wordSize);

   ld(FP, SP, - 2 * wordSize);

 #endif

 }

 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) {

   // determine java_thread register

   if (!java_thread->is_valid()) {

 #ifndef OPT_THREAD

     java_thread = T1;

     get_thread(java_thread);

 #else

     java_thread = TREG;

 #endif

   }

   // we must set sp to zero to clear frame

   st_ptr(R0, java_thread, in_bytes(JavaThread::last_Java_sp_offset()));

   // must clear fp, so that compiled frames are not confused; it is possible

   // that we need it only for debugging

   if(clear_fp) {

     st_ptr(R0, java_thread, in_bytes(JavaThread::last_Java_fp_offset()));

   }

   // Always clear the pc because it could have been set by make_walkable()

   st_ptr(R0, java_thread, in_bytes(JavaThread::last_Java_pc_offset()));

 }

 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {

   Register thread = TREG;

 #ifndef OPT_THREAD

   get_thread(thread);

 #endif

   // we must set sp to zero to clear frame

   sd(R0, Address(thread, JavaThread::last_Java_sp_offset()));

   // must clear fp, so that compiled frames are not confused; it is

   // possible that we need it only for debugging

   if (clear_fp) {

     sd(R0, Address(thread, JavaThread::last_Java_fp_offset()));

   }

   // Always clear the pc because it could have been set by make_walkable()

   sd(R0, Address(thread, JavaThread::last_Java_pc_offset()));

 }

 // Write serialization page so VM thread can do a pseudo remote membar.

 // We use the current thread pointer to calculate a thread specific

 // offset to write to within the page. This minimizes bus traffic

 // due to cache line collision.

 void MacroAssembler::serialize_memory(Register thread, Register tmp) {

   move(tmp, thread);

   srl(tmp, tmp,os::get_serialize_page_shift_count());

   move(AT, (os::vm_page_size() - sizeof(int)));

   andr(tmp, tmp,AT);

   sw(tmp,Address(tmp, (intptr_t)os::get_memory_serialize_page()));

 }

 // Calls to C land

 //

 // When entering C land, the fp, & sp of the last Java frame have to be recorded

 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp

 // has to be reset to 0. This is required to allow proper stack traversal.

 void MacroAssembler::set_last_Java_frame(Register java_thread,

                                          Register last_java_sp,

                                          Register last_java_fp,

                                          address  last_java_pc) {

   // determine java_thread register

   if (!java_thread->is_valid()) {

 #ifndef OPT_THREAD

     java_thread = T2;

     get_thread(java_thread);

 #else

     java_thread = TREG;

 #endif

   }

   // determine last_java_sp register

   if (!last_java_sp->is_valid()) {

     last_java_sp = SP;

   }

   // last_java_fp is optional

   if (last_java_fp->is_valid()) {

     st_ptr(last_java_fp, java_thread, in_bytes(JavaThread::last_Java_fp_offset()));

   }

   // last_java_pc is optional

   if (last_java_pc != NULL) {

     relocate(relocInfo::internal_word_type);

     patchable_set48(AT, (long)last_java_pc);

     st_ptr(AT, java_thread, in_bytes(JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()));

   }

   st_ptr(last_java_sp, java_thread, in_bytes(JavaThread::last_Java_sp_offset()));

 }

 void MacroAssembler::set_last_Java_frame(Register last_java_sp,

                                          Register last_java_fp,

                                          address  last_java_pc) {

   // determine last_java_sp register

   if (!last_java_sp->is_valid()) {

     last_java_sp = SP;

   }

   Register thread = TREG;

 #ifndef OPT_THREAD

   get_thread(thread);

 #endif

   // last_java_fp is optional

   if (last_java_fp->is_valid()) {

     sd(last_java_fp, Address(thread, JavaThread::last_Java_fp_offset()));

   }

   // last_java_pc is optional

   if (last_java_pc != NULL) {

     relocate(relocInfo::internal_word_type);

     patchable_set48(AT, (long)last_java_pc);

     st_ptr(AT, thread, in_bytes(JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()));

   }

   sd(last_java_sp, Address(thread, JavaThread::last_Java_sp_offset()));

 }

 //////////////////////////////////////////////////////////////////////////////////

 #if INCLUDE_ALL_GCS

 void MacroAssembler::g1_write_barrier_pre(Register obj,

                                           Register pre_val,

                                           Register thread,

                                           Register tmp,

                                           bool tosca_live,

                                           bool expand_call) {

   // If expand_call is true then we expand the call_VM_leaf macro

   // directly to skip generating the check by

   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.

 #ifdef _LP64

   assert(thread == TREG, "must be");

 #endif // _LP64

   Label done;

   Label runtime;

   assert(pre_val != noreg, "check this code");

   if (obj != noreg) {

     assert_different_registers(obj, pre_val, tmp);

     assert(pre_val != V0, "check this code");

   }

   Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +

                                        PtrQueue::byte_offset_of_active()));

   Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +

                                        PtrQueue::byte_offset_of_index()));

   Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +

                                        PtrQueue::byte_offset_of_buf()));

   // Is marking active?

   if (in_bytes(PtrQueue::byte_width_of_active()) == 4) {

     lw(AT, in_progress);

   } else {

     assert(in_bytes(PtrQueue::byte_width_of_active()) == 1, "Assumption");

     lb(AT, in_progress);

   }

   beq(AT, R0, done);

   delayed()->nop();

   // Do we need to load the previous value?

   if (obj != noreg) {

     load_heap_oop(pre_val, Address(obj, 0));

   }

   // Is the previous value null?

   beq(pre_val, R0, done);

   delayed()->nop();

   // Can we store original value in the thread's buffer?

   // Is index == 0?

   // (The index field is typed as size_t.)

   ld(tmp, index);

   beq(tmp, R0, runtime);

   delayed()->nop();

   daddiu(tmp, tmp, -1 * wordSize);

   sd(tmp, index);

   ld(AT, buffer);

   daddu(tmp, tmp, AT);

   // Record the previous value

   sd(pre_val, tmp, 0);

   beq(R0, R0, done);

   delayed()->nop();

   bind(runtime);

   // save the live input values

   if (tosca_live) push(V0);

   if (obj != noreg && obj != V0) push(obj);

   if (pre_val != V0) push(pre_val);

   // Calling the runtime using the regular call_VM_leaf mechanism generates

   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)

   // that checks that the *(fp+frame::interpreter_frame_last_sp) == NULL.

   //

   // If we care generating the pre-barrier without a frame (e.g. in the

   // intrinsified Reference.get() routine) then fp might be pointing to

   // the caller frame and so this check will most likely fail at runtime.

   //

   // Expanding the call directly bypasses the generation of the check.

   // So when we do not have have a full interpreter frame on the stack

   // expand_call should be passed true.

   NOT_LP64( push(thread); )

   if (expand_call) {

     LP64_ONLY( assert(pre_val != A1, "smashed arg"); )

     if (thread != A1) move(A1, thread);

     if (pre_val != A0) move(A0, pre_val);

     MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2);

   } else {

     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);

   }

   NOT_LP64( pop(thread); )

   // save the live input values

   if (pre_val != V0)

     pop(pre_val);

   if (obj != noreg && obj != V0)

     pop(obj);

   if(tosca_live) pop(V0);

   bind(done);

 }

 void MacroAssembler::g1_write_barrier_post(Register store_addr,

                                            Register new_val,

                                            Register thread,

                                            Register tmp,

                                            Register tmp2) {

   assert(tmp  != AT, "must be");

   assert(tmp2 != AT, "must be");

 #ifdef _LP64

   assert(thread == TREG, "must be");

 #endif // _LP64

   Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +

                                        PtrQueue::byte_offset_of_index()));

   Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +

                                        PtrQueue::byte_offset_of_buf()));

   BarrierSet* bs = Universe::heap()->barrier_set();

   CardTableModRefBS* ct = (CardTableModRefBS*)bs;

   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");

   Label done;

   Label runtime;

   // Does store cross heap regions?

   xorr(AT, store_addr, new_val);

   dsrl(AT, AT, HeapRegion::LogOfHRGrainBytes);

   beq(AT, R0, done);

   delayed()->nop();

   // crosses regions, storing NULL?

   beq(new_val, R0, done);

   delayed()->nop();

   // storing region crossing non-NULL, is card already dirty?

   const Register card_addr = tmp;

   const Register cardtable = tmp2;

   move(card_addr, store_addr);

   dsrl(card_addr, card_addr, CardTableModRefBS::card_shift);

   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT

   // a valid address and therefore is not properly handled by the relocation code.

   set64(cardtable, (intptr_t)ct->byte_map_base);

   daddu(card_addr, card_addr, cardtable);

   lb(AT, card_addr, 0);

   daddiu(AT, AT, -1 * (int)G1SATBCardTableModRefBS::g1_young_card_val());

   beq(AT, R0, done);

   delayed()->nop();

   sync();

   lb(AT, card_addr, 0);

   daddiu(AT, AT, -1 * (int)(int)CardTableModRefBS::dirty_card_val());

   beq(AT, R0, done);

   delayed()->nop();

   // storing a region crossing, non-NULL oop, card is clean.

   // dirty card and log.

   move(AT, (int)CardTableModRefBS::dirty_card_val());

   sb(AT, card_addr, 0);

   lw(AT, queue_index);

   beq(AT, R0, runtime);

   delayed()->nop();

   daddiu(AT, AT, -1 * wordSize);

   sw(AT, queue_index);

   ld(tmp2, buffer);

 #ifdef _LP64

   ld(AT, queue_index);

   daddu(tmp2, tmp2, AT);

   sd(card_addr, tmp2, 0);

 #else

   lw(AT, queue_index);

   addu32(tmp2, tmp2, AT);

   sw(card_addr, tmp2, 0);

 #endif

   beq(R0, R0, done);

   delayed()->nop();

   bind(runtime);

   // save the live input values

   push(store_addr);

   push(new_val);

 #ifdef _LP64

   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, TREG);

 #else

   push(thread);

   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);

   pop(thread);

 #endif

   pop(new_val);

   pop(store_addr);

   bind(done);

 }

 #endif // INCLUDE_ALL_GCS

 //////////////////////////////////////////////////////////////////////////////////

 void MacroAssembler::store_check(Register obj) {

   // Does a store check for the oop in register obj. The content of

   // register obj is destroyed afterwards.

   store_check_part_1(obj);

   store_check_part_2(obj);

 }

 void MacroAssembler::store_check(Register obj, Address dst) {

   store_check(obj);

 }

 // split the store check operation so that other instructions can be scheduled inbetween

 void MacroAssembler::store_check_part_1(Register obj) {

   BarrierSet* bs = Universe::heap()->barrier_set();

   assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");

 #ifdef _LP64

   dsrl(obj, obj, CardTableModRefBS::card_shift);

 #else

   shr(obj, CardTableModRefBS::card_shift);

 #endif

 }

 void MacroAssembler::store_check_part_2(Register obj) {

   BarrierSet* bs = Universe::heap()->barrier_set();

   assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");

   CardTableModRefBS* ct = (CardTableModRefBS*)bs;

   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");

   set64(AT, (long)ct->byte_map_base);

 #ifdef _LP64

   dadd(AT, AT, obj);

 #else

   add(AT, AT, obj);

 #endif

   if (UseConcMarkSweepGC) sync();

   sb(R0, AT, 0);

 }

 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.

 void MacroAssembler::tlab_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes,

                                    Register t1, Register t2, Label& slow_case) {

   assert_different_registers(obj, var_size_in_bytes, t1, t2, AT);

   Register end = t2;

 #ifndef OPT_THREAD

   Register thread = t1;

   get_thread(thread);

 #else

   Register thread = TREG;

 #endif

   verify_tlab(t1, t2);//blows t1&t2

   ld_ptr(obj, thread, in_bytes(JavaThread::tlab_top_offset()));

   if (var_size_in_bytes == NOREG) {

     set64(AT, con_size_in_bytes);

     add(end, obj, AT);

   } else {

     add(end, obj, var_size_in_bytes);

   }

   ld_ptr(AT, thread, in_bytes(JavaThread::tlab_end_offset()));

   sltu(AT, AT, end);

   bne_far(AT, R0, slow_case);

   delayed()->nop();

   // update the tlab top pointer

   st_ptr(end, thread, in_bytes(JavaThread::tlab_top_offset()));

   verify_tlab(t1, t2);

 }

 // Defines obj, preserves var_size_in_bytes

 void MacroAssembler::eden_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes,

                                    Register t1, Register t2, Label& slow_case) {

   assert_different_registers(obj, var_size_in_bytes, t1, AT);

   if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {

     // No allocation in the shared eden.

     b_far(slow_case);

     delayed()->nop();

   } else {

 #ifndef _LP64

     Address heap_top(t1, Assembler::split_low((intptr_t)Universe::heap()->top_addr()));

     lui(t1, split_high((intptr_t)Universe::heap()->top_addr()));

 #else

     Address heap_top(t1);

     li(t1, (long)Universe::heap()->top_addr());

 #endif

     ld_ptr(obj, heap_top);

     Register end = t2;

     Label retry;

     bind(retry);

     if (var_size_in_bytes == NOREG) {

       set64(AT, con_size_in_bytes);

       add(end, obj, AT);

     } else {

       add(end, obj, var_size_in_bytes);

     }

     // if end < obj then we wrapped around => object too long => slow case

     sltu(AT, end, obj);

     bne_far(AT, R0, slow_case);

     delayed()->nop();

     li(AT, (long)Universe::heap()->end_addr());

     ld_ptr(AT, AT, 0);

     sltu(AT, AT, end);

     bne_far(AT, R0, slow_case);

     delayed()->nop();

     // Compare obj with the top addr, and if still equal, store the new top addr in

     // end at the address of the top addr pointer. Sets ZF if was equal, and clears

     // it otherwise. Use lock prefix for atomicity on MPs.

     //if (os::is_MP()) {

     //  sync();

     //}

     // if someone beat us on the allocation, try again, otherwise continue

     cmpxchg(end, heap_top, obj);

     beq_far(AT, R0, retry);

     delayed()->nop();

   }

 }

 // C2 doesn't invoke this one.

 void MacroAssembler::tlab_refill(Label& retry, Label& try_eden, Label& slow_case) {

   Register top = T0;

   Register t1  = T1;

   Register t2  = T9;

   Register t3  = T3;

   Register thread_reg = T8;

   assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ T2, A4);

   Label do_refill, discard_tlab;

   if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {

     // No allocation in the shared eden.

     b(slow_case);

     delayed()->nop();

   }

   get_thread(thread_reg);

   ld_ptr(top, thread_reg, in_bytes(JavaThread::tlab_top_offset()));

   ld_ptr(t1,  thread_reg, in_bytes(JavaThread::tlab_end_offset()));

   // calculate amount of free space

   sub(t1, t1, top);

   shr(t1, LogHeapWordSize);

   // Retain tlab and allocate object in shared space if

   // the amount free in the tlab is too large to discard.

   ld_ptr(t2, thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset()));

   slt(AT, t2, t1);

   beq(AT, R0, discard_tlab);

   delayed()->nop();

   // Retain

 #ifndef _LP64

   move(AT, ThreadLocalAllocBuffer::refill_waste_limit_increment());

 #else

   li(AT, ThreadLocalAllocBuffer::refill_waste_limit_increment());

 #endif

   add(t2, t2, AT);

   st_ptr(t2, thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset()));

   if (TLABStats) {

     // increment number of slow_allocations

     lw(AT, thread_reg, in_bytes(JavaThread::tlab_slow_allocations_offset()));

     addiu(AT, AT, 1);

     sw(AT, thread_reg, in_bytes(JavaThread::tlab_slow_allocations_offset()));

   }

   b(try_eden);

   delayed()->nop();

   bind(discard_tlab);

   if (TLABStats) {

     // increment number of refills

     lw(AT, thread_reg, in_bytes(JavaThread::tlab_number_of_refills_offset()));

     addi(AT, AT, 1);

     sw(AT, thread_reg, in_bytes(JavaThread::tlab_number_of_refills_offset()));

     // accumulate wastage -- t1 is amount free in tlab

     lw(AT, thread_reg, in_bytes(JavaThread::tlab_fast_refill_waste_offset()));

     add(AT, AT, t1);

     sw(AT, thread_reg, in_bytes(JavaThread::tlab_fast_refill_waste_offset()));

   }

   // if tlab is currently allocated (top or end != null) then

   // fill [top, end + alignment_reserve) with array object

   beq(top, R0, do_refill);

   delayed()->nop();

   // set up the mark word

   li(AT, (long)markOopDesc::prototype()->copy_set_hash(0x2));

   st_ptr(AT, top, oopDesc::mark_offset_in_bytes());

   // set the length to the remaining space

   addi(t1, t1, - typeArrayOopDesc::header_size(T_INT));

   addi(t1, t1, ThreadLocalAllocBuffer::alignment_reserve());

   shl(t1, log2_intptr(HeapWordSize/sizeof(jint)));

   sw(t1, top, arrayOopDesc::length_offset_in_bytes());

   // set klass to intArrayKlass

 #ifndef _LP64

   lui(AT, split_high((intptr_t)Universe::intArrayKlassObj_addr()));

   lw(t1, AT, split_low((intptr_t)Universe::intArrayKlassObj_addr()));

 #else

   li(AT, (intptr_t)Universe::intArrayKlassObj_addr());

   ld_ptr(t1, AT, 0);

 #endif

   //st_ptr(t1, top, oopDesc::klass_offset_in_bytes());

   store_klass(top, t1);

   ld_ptr(t1, thread_reg, in_bytes(JavaThread::tlab_start_offset()));

   subu(t1, top, t1);

   incr_allocated_bytes(thread_reg, t1, 0);

   // refill the tlab with an eden allocation

   bind(do_refill);

   ld_ptr(t1, thread_reg, in_bytes(JavaThread::tlab_size_offset()));

   shl(t1, LogHeapWordSize);

   // add object_size ??

   eden_allocate(top, t1, 0, t2, t3, slow_case);

   // Check that t1 was preserved in eden_allocate.

 #ifdef ASSERT

   if (UseTLAB) {

     Label ok;

     assert_different_registers(thread_reg, t1);

     ld_ptr(AT, thread_reg, in_bytes(JavaThread::tlab_size_offset()));

     shl(AT, LogHeapWordSize);

     beq(AT, t1, ok);

     delayed()->nop();

     stop("assert(t1 != tlab size)");

     should_not_reach_here();

     bind(ok);

   }

 #endif

   st_ptr(top, thread_reg, in_bytes(JavaThread::tlab_start_offset()));

   st_ptr(top, thread_reg, in_bytes(JavaThread::tlab_top_offset()));

   add(top, top, t1);

   addi(top, top, - ThreadLocalAllocBuffer::alignment_reserve_in_bytes());

   st_ptr(top, thread_reg, in_bytes(JavaThread::tlab_end_offset()));

   verify_tlab(t1, t2);

   b(retry);

   delayed()->nop();

 }

 void MacroAssembler::incr_allocated_bytes(Register thread,

                                           Register var_size_in_bytes,

                                           int con_size_in_bytes,

                                           Register t1) {

   if (!thread->is_valid()) {

 #ifndef OPT_THREAD

     assert(t1->is_valid(), "need temp reg");

     thread = t1;

     get_thread(thread);

 #else

     thread = TREG;

 #endif

   }

   ld_ptr(AT, thread, in_bytes(JavaThread::allocated_bytes_offset()));

   if (var_size_in_bytes->is_valid()) {

     addu(AT, AT, var_size_in_bytes);

   } else {

     addiu(AT, AT, con_size_in_bytes);

   }

   st_ptr(AT, thread, in_bytes(JavaThread::allocated_bytes_offset()));

 }

 static const double     pi_4 =  0.7853981633974483;

 // must get argument(a double) in F12/F13

 //void MacroAssembler::trigfunc(char trig, bool preserve_cpu_regs, int num_fpu_regs_in_use) {

 //We need to preseve the register which maybe modified during the Call

 void MacroAssembler::trigfunc(char trig, int num_fpu_regs_in_use) {

   // save all modified register here

   // FIXME, in the disassembly of tirgfunc, only used V0, V1, T9, SP, RA, so we ony save V0, V1, T9

   pushad();

   // we should preserve the stack space before we call

   addi(SP, SP, -wordSize * 2);

   switch (trig){

     case 's' :

       call( CAST_FROM_FN_PTR(address, SharedRuntime::dsin), relocInfo::runtime_call_type );

       delayed()->nop();

       break;

     case 'c':

       call( CAST_FROM_FN_PTR(address, SharedRuntime::dcos), relocInfo::runtime_call_type );

       delayed()->nop();

       break;

     case 't':

       call( CAST_FROM_FN_PTR(address, SharedRuntime::dtan), relocInfo::runtime_call_type );

       delayed()->nop();

       break;

     default:assert (false, "bad intrinsic");

     break;

   }

   addi(SP, SP, wordSize * 2);

   popad();

 }

 #ifdef _LP64

 void MacroAssembler::li(Register rd, long imm) {

   if (imm <= max_jint && imm >= min_jint) {

     li32(rd, (int)imm);

   } else if (julong(imm) <= 0xFFFFFFFF) {

     assert_not_delayed();

     // lui sign-extends, so we can't use that.

     ori(rd, R0, julong(imm) >> 16);

     dsll(rd, rd, 16);

     ori(rd, rd, split_low(imm));

   } else if ((imm > 0) && is_simm16(imm >> 32)) {

     // A 48-bit address

     li48(rd, imm);

   } else {

     li64(rd, imm);

   }

 }

 #else

 void MacroAssembler::li(Register rd, long imm) {

   li32(rd, (int)imm);

 }

 #endif

 void MacroAssembler::li32(Register reg, int imm) {

   if (is_simm16(imm)) {

     // for imm < 0, we should use addi instead of addiu.

     //

     //  java.lang.StringCoding$StringDecoder.decode(jobject, jint, jint)

     //

     //  78 move [int:-1|I] [a0|I]

     //    : daddi a0, zero, 0xffffffff  (correct)

     //    : daddiu a0, zero, 0xffffffff (incorrect)

     //

     if (imm >= 0)

       addiu(reg, R0, imm);

     else

       addi(reg, R0, imm);

   } else {

     lui(reg, split_low(imm >> 16));

     if (split_low(imm))

       ori(reg, reg, split_low(imm));

   }

 }

 #ifdef _LP64

 void MacroAssembler::set64(Register d, jlong value) {

   assert_not_delayed();

   int hi = (int)(value >> 32);

   int lo = (int)(value & ~0);

   if (value == lo) {  // 32-bit integer

     if (is_simm16(value)) {

       daddiu(d, R0, value);

     } else {

       lui(d, split_low(value >> 16));

       if (split_low(value)) {

         ori(d, d, split_low(value));

       }

     }

   } else if (hi == 0) {  // hardware zero-extends to upper 32

       ori(d, R0, julong(value) >> 16);

       dsll(d, d, 16);

       if (split_low(value)) {

         ori(d, d, split_low(value));

       }

   } else if ((value> 0) && is_simm16(value >> 32)) {  // li48

     // 4 insts

     li48(d, value);

   } else {  // li64

     // 6 insts

     li64(d, value);

   }

 }

 int MacroAssembler::insts_for_set64(jlong value) {

   int hi = (int)(value >> 32);

   int lo = (int)(value & ~0);

   int count = 0;

   if (value == lo) {  // 32-bit integer

     if (is_simm16(value)) {

       //daddiu(d, R0, value);

       count++;

     } else {

       //lui(d, split_low(value >> 16));

       count++;

       if (split_low(value)) {

         //ori(d, d, split_low(value));

         count++;

       }

     }

   } else if (hi == 0) {  // hardware zero-extends to upper 32

       //ori(d, R0, julong(value) >> 16);

       //dsll(d, d, 16);

       count += 2;

       if (split_low(value)) {

         //ori(d, d, split_low(value));

         count++;

       }

   } else if ((value> 0) && is_simm16(value >> 32)) {  // li48

     // 4 insts

     //li48(d, value);

     count += 4;

   } else {  // li64

     // 6 insts

     //li64(d, value);

     count += 6;

   }

   return count;

 }

 void MacroAssembler::patchable_set48(Register d, jlong value) {

   assert_not_delayed();

   int hi = (int)(value >> 32);

   int lo = (int)(value & ~0);

   int count = 0;

   if (value == lo) {  // 32-bit integer

     if (is_simm16(value)) {

       daddiu(d, R0, value);

       count += 1;

     } else {

       lui(d, split_low(value >> 16));

       count += 1;

       if (split_low(value)) {

         ori(d, d, split_low(value));

         count += 1;

       }

     }

   } else if (hi == 0) {  // hardware zero-extends to upper 32

       ori(d, R0, julong(value) >> 16);

       dsll(d, d, 16);

       count += 2;

       if (split_low(value)) {

         ori(d, d, split_low(value));

         count += 1;

       }

   } else if ((value> 0) && is_simm16(value >> 32)) {  // li48

     // 4 insts

     li48(d, value);

     count += 4;

   } else {  // li64

     tty->print_cr("value = 0x%x", value);

     guarantee(false, "Not supported yet !");

   }

   while (count < 4) {

     nop();

     count++;

   }

 }

 void MacroAssembler::patchable_set32(Register d, jlong value) {

   assert_not_delayed();

   int hi = (int)(value >> 32);

   int lo = (int)(value & ~0);

   int count = 0;

   if (value == lo) {  // 32-bit integer

     if (is_simm16(value)) {

       daddiu(d, R0, value);

       count += 1;

     } else {

       lui(d, split_low(value >> 16));

       count += 1;

       if (split_low(value)) {

         ori(d, d, split_low(value));

         count += 1;

       }

     }

   } else if (hi == 0) {  // hardware zero-extends to upper 32

       ori(d, R0, julong(value) >> 16);

       dsll(d, d, 16);

       count += 2;

       if (split_low(value)) {

         ori(d, d, split_low(value));

         count += 1;

       }

   } else {

     tty->print_cr("value = 0x%x", value);

     guarantee(false, "Not supported yet !");

   }

   while (count < 3) {

     nop();

     count++;

   }

 }

 void MacroAssembler::patchable_call32(Register d, jlong value) {

   assert_not_delayed();

   int hi = (int)(value >> 32);

   int lo = (int)(value & ~0);

   int count = 0;

   if (value == lo) {  // 32-bit integer

     if (is_simm16(value)) {

       daddiu(d, R0, value);

       count += 1;

     } else {

       lui(d, split_low(value >> 16));

       count += 1;

       if (split_low(value)) {

         ori(d, d, split_low(value));

         count += 1;

       }

     }

   } else {

     tty->print_cr("value = 0x%x", value);

     guarantee(false, "Not supported yet !");

   }

   while (count < 2) {

     nop();

     count++;

   }

 }

 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {

   assert(UseCompressedClassPointers, "should only be used for compressed header");

   assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");

   int klass_index = oop_recorder()->find_index(k);

   RelocationHolder rspec = metadata_Relocation::spec(klass_index);

   long narrowKlass = (long)Klass::encode_klass(k);

   relocate(rspec, Assembler::narrow_oop_operand);

   patchable_set48(dst, narrowKlass);

 }

 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {

   assert(UseCompressedOops, "should only be used for compressed header");

   assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");

   int oop_index = oop_recorder()->find_index(obj);

   RelocationHolder rspec = oop_Relocation::spec(oop_index);

   relocate(rspec, Assembler::narrow_oop_operand);

   patchable_set48(dst, oop_index);

 }

 void MacroAssembler::li64(Register rd, long imm) {

   assert_not_delayed();

   lui(rd, split_low(imm >> 48));

   ori(rd, rd, split_low(imm >> 32));

   dsll(rd, rd, 16);

   ori(rd, rd, split_low(imm >> 16));

   dsll(rd, rd, 16);

   ori(rd, rd, split_low(imm));

 }

 void MacroAssembler::li48(Register rd, long imm) {

   assert_not_delayed();

   assert(is_simm16(imm >> 32), "Not a 48-bit address");

   lui(rd, imm >> 32);

   ori(rd, rd, split_low(imm >> 16));

   dsll(rd, rd, 16);

   ori(rd, rd, split_low(imm));

 }

 #endif

 void MacroAssembler::verify_oop(Register reg, const char* s) {

   if (!VerifyOops) return;

   const char * b = NULL;

   stringStream ss;

   ss.print("verify_oop: %s: %s", reg->name(), s);

   b = code_string(ss.as_string());

 #ifdef _LP64

   pushad();

   move(A1, reg);

   li(A0, (long)b);

   li(AT, (long)StubRoutines::verify_oop_subroutine_entry_address());

   ld(T9, AT, 0);

   jalr(T9);

   delayed()->nop();

   popad();

 #else

   // Pass register number to verify_oop_subroutine

   sw(T0, SP, - wordSize);

   sw(T1, SP, - 2*wordSize);

   sw(RA, SP, - 3*wordSize);

   sw(A0, SP ,- 4*wordSize);

   sw(A1, SP ,- 5*wordSize);

   sw(AT, SP ,- 6*wordSize);

   sw(T9, SP ,- 7*wordSize);

   addiu(SP, SP, - 7 * wordSize);

   move(A1, reg);

   li(A0, (long)b);

   // call indirectly to solve generation ordering problem

   li(AT, (long)StubRoutines::verify_oop_subroutine_entry_address());

   lw(T9, AT, 0);

   jalr(T9);

   delayed()->nop();

   lw(T0, SP, 6* wordSize);

   lw(T1, SP, 5* wordSize);

   lw(RA, SP, 4* wordSize);

   lw(A0, SP, 3* wordSize);

   lw(A1, SP, 2* wordSize);

   lw(AT, SP, 1* wordSize);

   lw(T9, SP, 0* wordSize);

   addiu(SP, SP, 7 * wordSize);

 #endif

 }

 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {

   if (!VerifyOops) {

     nop();

     return;

   }

   // Pass register number to verify_oop_subroutine

   const char * b = NULL;

   stringStream ss;

   ss.print("verify_oop_addr: %s",  s);

   b = code_string(ss.as_string());

   st_ptr(T0, SP, - wordSize);

   st_ptr(T1, SP, - 2*wordSize);

   st_ptr(RA, SP, - 3*wordSize);

   st_ptr(A0, SP, - 4*wordSize);

   st_ptr(A1, SP, - 5*wordSize);

   st_ptr(AT, SP, - 6*wordSize);

   st_ptr(T9, SP, - 7*wordSize);

   ld_ptr(A1, addr);   // addr may use SP, so load from it before change SP

   addiu(SP, SP, - 7 * wordSize);

   li(A0, (long)b);

   // call indirectly to solve generation ordering problem

   li(AT, (long)StubRoutines::verify_oop_subroutine_entry_address());

   ld_ptr(T9, AT, 0);

   jalr(T9);

   delayed()->nop();

   ld_ptr(T0, SP, 6* wordSize);

   ld_ptr(T1, SP, 5* wordSize);

   ld_ptr(RA, SP, 4* wordSize);

   ld_ptr(A0, SP, 3* wordSize);

   ld_ptr(A1, SP, 2* wordSize);

   ld_ptr(AT, SP, 1* wordSize);

   ld_ptr(T9, SP, 0* wordSize);

   addiu(SP, SP, 7 * wordSize);

 }

 // used registers :  T0, T1

 void MacroAssembler::verify_oop_subroutine() {

   // RA: ra

   // A0: char* error message

   // A1: oop   object to verify

   Label exit, error;

   // increment counter

   li(T0, (long)StubRoutines::verify_oop_count_addr());

   lw(AT, T0, 0);

 #ifdef _LP64

   daddi(AT, AT, 1);

 #else

   addi(AT, AT, 1);

 #endif

   sw(AT, T0, 0);

   // make sure object is 'reasonable'

   beq(A1, R0, exit);         // if obj is NULL it is ok

   delayed()->nop();

   // Check if the oop is in the right area of memory

   // const int oop_mask = Universe::verify_oop_mask();

   // const int oop_bits = Universe::verify_oop_bits();

   const uintptr_t oop_mask = Universe::verify_oop_mask();

   const uintptr_t oop_bits = Universe::verify_oop_bits();

   li(AT, oop_mask);

   andr(T0, A1, AT);

   li(AT, oop_bits);

   bne(T0, AT, error);

   delayed()->nop();

   // make sure klass is 'reasonable'

   // add for compressedoops

   reinit_heapbase();

   // add for compressedoops

   load_klass(T0, A1);

   beq(T0, R0, error);                        // if klass is NULL it is broken

   delayed()->nop();

   // return if everything seems ok

   bind(exit);

   jr(RA);

   delayed()->nop();

   // handle errors

   bind(error);

   pushad();

 #ifndef _LP64

   addi(SP, SP, (-1) * wordSize);

 #endif

   call(CAST_FROM_FN_PTR(address, MacroAssembler::debug), relocInfo::runtime_call_type);

   delayed()->nop();

 #ifndef _LP64

   addiu(SP, SP, 1 * wordSize);

 #endif

   popad();

   jr(RA);

   delayed()->nop();

 }

 void MacroAssembler::verify_tlab(Register t1, Register t2) {

 #ifdef ASSERT

   assert_different_registers(t1, t2, AT);

   if (UseTLAB && VerifyOops) {

     Label next, ok;

     get_thread(t1);

     ld_ptr(t2, t1, in_bytes(JavaThread::tlab_top_offset()));

     ld_ptr(AT, t1, in_bytes(JavaThread::tlab_start_offset()));

     sltu(AT, t2, AT);

     beq(AT, R0, next);

     delayed()->nop();

     stop("assert(top >= start)");

     bind(next);

     ld_ptr(AT, t1, in_bytes(JavaThread::tlab_end_offset()));

     sltu(AT, AT, t2);

     beq(AT, R0, ok);

     delayed()->nop();

     stop("assert(top <= end)");

     bind(ok);

   }

 #endif

 }

 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,

                                                        Register tmp,

                                                        int offset) {

   intptr_t value = *delayed_value_addr;

   if (value != 0)

   return RegisterOrConstant(value + offset);

   AddressLiteral a(delayed_value_addr);

   // load indirectly to solve generation ordering problem

   //movptr(tmp, ExternalAddress((address) delayed_value_addr));

   //ld(tmp, a);

   if (offset != 0)

     daddi(tmp,tmp, offset);

   return RegisterOrConstant(tmp);

 }

 void MacroAssembler::hswap(Register reg) {

   //short

   //andi(reg, reg, 0xffff);

   srl(AT, reg, 8);

   sll(reg, reg, 24);

   sra(reg, reg, 16);

   orr(reg, reg, AT);

 }

 void MacroAssembler::huswap(Register reg) {

 #ifdef _LP64

   dsrl(AT, reg, 8);

   dsll(reg, reg, 24);

   dsrl(reg, reg, 16);

   orr(reg, reg, AT);

   andi(reg, reg, 0xffff);

 #else

   //andi(reg, reg, 0xffff);

   srl(AT, reg, 8);

   sll(reg, reg, 24);

   srl(reg, reg, 16);

   orr(reg, reg, AT);

 #endif

 }

 // something funny to do this will only one more register AT

 // 32 bits

 void MacroAssembler::swap(Register reg) {

   srl(AT, reg, 8);

   sll(reg, reg, 24);

   orr(reg, reg, AT);

   //reg : 4 1 2 3

   srl(AT, AT, 16);

   xorr(AT, AT, reg);

   andi(AT, AT, 0xff);

   //AT : 0 0 0 1^3);

   xorr(reg, reg, AT);

   //reg : 4 1 2 1

   sll(AT, AT, 16);

   xorr(reg, reg, AT);

   //reg : 4 3 2 1

 }

 #ifdef _LP64

 // do 32-bit CAS using MIPS64 lld/scd

 //

 //  cas_int should only compare 32-bits of the memory value.

 //  However, lld/scd will do 64-bit operation, which violates the intention of cas_int.

 //  To simulate a 32-bit atomic operation, the value loaded with LLD should be split into

 //  tow halves, and only the low-32 bits is compared. If equals, the low-32 bits of newval,

 //  plus the high-32 bits or memory value, are stored togethor with SCD.

 //

 //Example:

 //

 //      double d = 3.1415926;

 //      System.err.println("hello" + d);

 //

 //  sun.misc.FloatingDecimal$1.<init>()

 //   |

 //   `- java.util.concurrent.atomic.AtomicInteger::compareAndSet()

 //

 //  38 cas_int [a7a7|J] [a0|I] [a6|I]

 //   a0: 0xffffffffe8ea9f63 pc: 0x55647f3354

 //   a6: 0x4ab325aa

 //

 //again:

 //   0x00000055647f3c5c: lld at, 0x0(a7)                          ; 64-bit load, "0xe8ea9f63"

 //

 //   0x00000055647f3c60: sll t9, at, 0                            ; t9: low-32 bits (sign extended)

 //   0x00000055647f3c64: dsrl32 t8, at, 0                         ; t8: high-32 bits

 //   0x00000055647f3c68: dsll32 t8, t8, 0

 //   0x00000055647f3c6c: bne t9, a0, 0x00000055647f3c9c           ; goto nequal

 //   0x00000055647f3c70: sll zero, zero, 0

 //

 //   0x00000055647f3c74: ori v1, zero, 0xffffffff                 ; v1: low-32 bits of newval (sign unextended)

 //   0x00000055647f3c78: dsll v1, v1, 16                          ; v1 = a6 & 0xFFFFFFFF;

 //   0x00000055647f3c7c: ori v1, v1, 0xffffffff

 //   0x00000055647f3c80: and v1, a6, v1

 //   0x00000055647f3c84: or at, t8, v1

 //   0x00000055647f3c88: scd at, 0x0(a7)

 //   0x00000055647f3c8c: beq at, zero, 0x00000055647f3c5c         ; goto again

 //   0x00000055647f3c90: sll zero, zero, 0

 //   0x00000055647f3c94: beq zero, zero, 0x00000055647f45ac       ; goto done

 //   0x00000055647f3c98: sll zero, zero, 0

 //nequal:

 //   0x00000055647f45a4: dadd a0, t9, zero

 //   0x00000055647f45a8: dadd at, zero, zero

 //done:

 //

 void MacroAssembler::cmpxchg32(Register x_reg, Address dest, Register c_reg) {

   // MIPS64 can use ll/sc for 32-bit atomic memory access

   Label done, again, nequal;

   bind(again);

   if (UseSyncLevel >= 10000 || UseSyncLevel == 1000 || UseSyncLevel == 4000) sync();

   ll(AT, dest);

   bne(AT, c_reg, nequal);

   delayed()->nop();

   move(AT, x_reg);

   sc(AT, dest);

   beq(AT, R0, again);

   delayed()->nop();

   b(done);

   delayed()->nop();

   // not xchged

   bind(nequal);

   sync();

   move(c_reg, AT);

   move(AT, R0);

   bind(done);

 }

 #endif  // cmpxchg32

 void MacroAssembler::cmpxchg(Register x_reg, Address dest, Register c_reg) {

   Label done, again, nequal;

   bind(again);

   if (UseSyncLevel >= 10000 || UseSyncLevel == 1000 || UseSyncLevel == 4000) sync();

 #ifdef _LP64

   lld(AT, dest);

 #else

   ll(AT, dest);

 #endif

   bne(AT, c_reg, nequal);

   delayed()->nop();

   move(AT, x_reg);

 #ifdef _LP64

   scd(AT, dest);

 #else

   sc(AT, dest);

 #endif

   beq(AT, R0, again);

   delayed()->nop();

   b(done);

   delayed()->nop();

   // not xchged

   bind(nequal);

   sync();

   move(c_reg, AT);

   move(AT, R0);

   bind(done);

 }

 void MacroAssembler::cmpxchg8(Register x_regLo, Register x_regHi, Address dest, Register c_regLo, Register c_regHi) {

   Label done, again, nequal;

   Register x_reg = x_regLo;

   dsll32(x_regHi, x_regHi, 0);

   dsll32(x_regLo, x_regLo, 0);

   dsrl32(x_regLo, x_regLo, 0);

   orr(x_reg, x_regLo, x_regHi);

   Register c_reg = c_regLo;

   dsll32(c_regHi, c_regHi, 0);

   dsll32(c_regLo, c_regLo, 0);

   dsrl32(c_regLo, c_regLo, 0);

   orr(c_reg, c_regLo, c_regHi);

   bind(again);

   if (UseSyncLevel >= 10000 || UseSyncLevel == 1000 || UseSyncLevel == 4000) sync();

   lld(AT, dest);

   bne(AT, c_reg, nequal);

   delayed()->nop();

   //move(AT, x_reg);

   dadd(AT, x_reg, R0);

   scd(AT, dest);

   beq(AT, R0, again);

   delayed()->nop();

   b(done);

   delayed()->nop();

   // not xchged

   bind(nequal);

   sync();

   //move(c_reg, AT);

   //move(AT, R0);

   dadd(c_reg, AT, R0);

   dadd(AT, R0, R0);

   bind(done);

 }

 // be sure the three register is different

 void MacroAssembler::rem_s(FloatRegister fd, FloatRegister fs, FloatRegister ft, FloatRegister tmp) {

   assert_different_registers(tmp, fs, ft);

   div_s(tmp, fs, ft);

   trunc_l_s(tmp, tmp);

   cvt_s_l(tmp, tmp);

   mul_s(tmp, tmp, ft);

   sub_s(fd, fs, tmp);

 }

 // be sure the three register is different

 void MacroAssembler::rem_d(FloatRegister fd, FloatRegister fs, FloatRegister ft, FloatRegister tmp) {

   assert_different_registers(tmp, fs, ft);

   div_d(tmp, fs, ft);

   trunc_l_d(tmp, tmp);

   cvt_d_l(tmp, tmp);

   mul_d(tmp, tmp, ft);

   sub_d(fd, fs, tmp);

 }

 // Fast_Lock and Fast_Unlock used by C2

 // Because the transitions from emitted code to the runtime

 // monitorenter/exit helper stubs are so slow it's critical that

 // we inline both the stack-locking fast-path and the inflated fast path.

 //

 // See also: cmpFastLock and cmpFastUnlock.

 //

 // What follows is a specialized inline transliteration of the code

 // in slow_enter() and slow_exit().  If we're concerned about I$ bloat

 // another option would be to emit TrySlowEnter and TrySlowExit methods