1.1 --- a/src/cpu/x86/vm/x86_64.ad Thu Mar 06 12:45:59 2014 +0400
1.2 +++ b/src/cpu/x86/vm/x86_64.ad Mon Feb 24 15:12:26 2014 -0800
1.3 @@ -2591,231 +2591,6 @@
1.4 %}
1.5
1.6
1.7 - // obj: object to lock
1.8 - // box: box address (header location) -- killed
1.9 - // tmp: rax -- killed
1.10 - // scr: rbx -- killed
1.11 - //
1.12 - // What follows is a direct transliteration of fast_lock() and fast_unlock()
1.13 - // from i486.ad. See that file for comments.
1.14 - // TODO: where possible switch from movq (r, 0) to movl(r,0) and
1.15 - // use the shorter encoding. (Movl clears the high-order 32-bits).
1.16 -
1.17 -
1.18 - enc_class Fast_Lock(rRegP obj, rRegP box, rax_RegI tmp, rRegP scr)
1.19 - %{
1.20 - Register objReg = as_Register((int)$obj$$reg);
1.21 - Register boxReg = as_Register((int)$box$$reg);
1.22 - Register tmpReg = as_Register($tmp$$reg);
1.23 - Register scrReg = as_Register($scr$$reg);
1.24 - MacroAssembler masm(&cbuf);
1.25 -
1.26 - // Verify uniqueness of register assignments -- necessary but not sufficient
1.27 - assert (objReg != boxReg && objReg != tmpReg &&
1.28 - objReg != scrReg && tmpReg != scrReg, "invariant") ;
1.29 -
1.30 - if (_counters != NULL) {
1.31 - masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
1.32 - }
1.33 - if (EmitSync & 1) {
1.34 - // Without cast to int32_t a movptr will destroy r10 which is typically obj
1.35 - masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
1.36 - masm.cmpptr(rsp, (int32_t)NULL_WORD) ;
1.37 - } else
1.38 - if (EmitSync & 2) {
1.39 - Label DONE_LABEL;
1.40 - if (UseBiasedLocking) {
1.41 - // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
1.42 - masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
1.43 - }
1.44 - // QQQ was movl...
1.45 - masm.movptr(tmpReg, 0x1);
1.46 - masm.orptr(tmpReg, Address(objReg, 0));
1.47 - masm.movptr(Address(boxReg, 0), tmpReg);
1.48 - if (os::is_MP()) {
1.49 - masm.lock();
1.50 - }
1.51 - masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
1.52 - masm.jcc(Assembler::equal, DONE_LABEL);
1.53 -
1.54 - // Recursive locking
1.55 - masm.subptr(tmpReg, rsp);
1.56 - masm.andptr(tmpReg, 7 - os::vm_page_size());
1.57 - masm.movptr(Address(boxReg, 0), tmpReg);
1.58 -
1.59 - masm.bind(DONE_LABEL);
1.60 - masm.nop(); // avoid branch to branch
1.61 - } else {
1.62 - Label DONE_LABEL, IsInflated, Egress;
1.63 -
1.64 - masm.movptr(tmpReg, Address(objReg, 0)) ;
1.65 - masm.testl (tmpReg, 0x02) ; // inflated vs stack-locked|neutral|biased
1.66 - masm.jcc (Assembler::notZero, IsInflated) ;
1.67 -
1.68 - // it's stack-locked, biased or neutral
1.69 - // TODO: optimize markword triage order to reduce the number of
1.70 - // conditional branches in the most common cases.
1.71 - // Beware -- there's a subtle invariant that fetch of the markword
1.72 - // at [FETCH], below, will never observe a biased encoding (*101b).
1.73 - // If this invariant is not held we'll suffer exclusion (safety) failure.
1.74 -
1.75 - if (UseBiasedLocking && !UseOptoBiasInlining) {
1.76 - masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters);
1.77 - masm.movptr(tmpReg, Address(objReg, 0)) ; // [FETCH]
1.78 - }
1.79 -
1.80 - // was q will it destroy high?
1.81 - masm.orl (tmpReg, 1) ;
1.82 - masm.movptr(Address(boxReg, 0), tmpReg) ;
1.83 - if (os::is_MP()) { masm.lock(); }
1.84 - masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
1.85 - if (_counters != NULL) {
1.86 - masm.cond_inc32(Assembler::equal,
1.87 - ExternalAddress((address) _counters->fast_path_entry_count_addr()));
1.88 - }
1.89 - masm.jcc (Assembler::equal, DONE_LABEL);
1.90 -
1.91 - // Recursive locking
1.92 - masm.subptr(tmpReg, rsp);
1.93 - masm.andptr(tmpReg, 7 - os::vm_page_size());
1.94 - masm.movptr(Address(boxReg, 0), tmpReg);
1.95 - if (_counters != NULL) {
1.96 - masm.cond_inc32(Assembler::equal,
1.97 - ExternalAddress((address) _counters->fast_path_entry_count_addr()));
1.98 - }
1.99 - masm.jmp (DONE_LABEL) ;
1.100 -
1.101 - masm.bind (IsInflated) ;
1.102 - // It's inflated
1.103 -
1.104 - // TODO: someday avoid the ST-before-CAS penalty by
1.105 - // relocating (deferring) the following ST.
1.106 - // We should also think about trying a CAS without having
1.107 - // fetched _owner. If the CAS is successful we may
1.108 - // avoid an RTO->RTS upgrade on the $line.
1.109 - // Without cast to int32_t a movptr will destroy r10 which is typically obj
1.110 - masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
1.111 -
1.112 - masm.mov (boxReg, tmpReg) ;
1.113 - masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
1.114 - masm.testptr(tmpReg, tmpReg) ;
1.115 - masm.jcc (Assembler::notZero, DONE_LABEL) ;
1.116 -
1.117 - // It's inflated and appears unlocked
1.118 - if (os::is_MP()) { masm.lock(); }
1.119 - masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
1.120 - // Intentional fall-through into DONE_LABEL ...
1.121 -
1.122 - masm.bind (DONE_LABEL) ;
1.123 - masm.nop () ; // avoid jmp to jmp
1.124 - }
1.125 - %}
1.126 -
1.127 - // obj: object to unlock
1.128 - // box: box address (displaced header location), killed
1.129 - // RBX: killed tmp; cannot be obj nor box
1.130 - enc_class Fast_Unlock(rRegP obj, rax_RegP box, rRegP tmp)
1.131 - %{
1.132 -
1.133 - Register objReg = as_Register($obj$$reg);
1.134 - Register boxReg = as_Register($box$$reg);
1.135 - Register tmpReg = as_Register($tmp$$reg);
1.136 - MacroAssembler masm(&cbuf);
1.137 -
1.138 - if (EmitSync & 4) {
1.139 - masm.cmpptr(rsp, 0) ;
1.140 - } else
1.141 - if (EmitSync & 8) {
1.142 - Label DONE_LABEL;
1.143 - if (UseBiasedLocking) {
1.144 - masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
1.145 - }
1.146 -
1.147 - // Check whether the displaced header is 0
1.148 - //(=> recursive unlock)
1.149 - masm.movptr(tmpReg, Address(boxReg, 0));
1.150 - masm.testptr(tmpReg, tmpReg);
1.151 - masm.jcc(Assembler::zero, DONE_LABEL);
1.152 -
1.153 - // If not recursive lock, reset the header to displaced header
1.154 - if (os::is_MP()) {
1.155 - masm.lock();
1.156 - }
1.157 - masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
1.158 - masm.bind(DONE_LABEL);
1.159 - masm.nop(); // avoid branch to branch
1.160 - } else {
1.161 - Label DONE_LABEL, Stacked, CheckSucc ;
1.162 -
1.163 - if (UseBiasedLocking && !UseOptoBiasInlining) {
1.164 - masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
1.165 - }
1.166 -
1.167 - masm.movptr(tmpReg, Address(objReg, 0)) ;
1.168 - masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ;
1.169 - masm.jcc (Assembler::zero, DONE_LABEL) ;
1.170 - masm.testl (tmpReg, 0x02) ;
1.171 - masm.jcc (Assembler::zero, Stacked) ;
1.172 -
1.173 - // It's inflated
1.174 - masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
1.175 - masm.xorptr(boxReg, r15_thread) ;
1.176 - masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
1.177 - masm.jcc (Assembler::notZero, DONE_LABEL) ;
1.178 - masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ;
1.179 - masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ;
1.180 - masm.jcc (Assembler::notZero, CheckSucc) ;
1.181 - masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
1.182 - masm.jmp (DONE_LABEL) ;
1.183 -
1.184 - if ((EmitSync & 65536) == 0) {
1.185 - Label LSuccess, LGoSlowPath ;
1.186 - masm.bind (CheckSucc) ;
1.187 - masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
1.188 - masm.jcc (Assembler::zero, LGoSlowPath) ;
1.189 -
1.190 - // I'd much rather use lock:andl m->_owner, 0 as it's faster than the
1.191 - // the explicit ST;MEMBAR combination, but masm doesn't currently support
1.192 - // "ANDQ M,IMM". Don't use MFENCE here. lock:add to TOS, xchg, etc
1.193 - // are all faster when the write buffer is populated.
1.194 - masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
1.195 - if (os::is_MP()) {
1.196 - masm.lock () ; masm.addl (Address(rsp, 0), 0) ;
1.197 - }
1.198 - masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
1.199 - masm.jcc (Assembler::notZero, LSuccess) ;
1.200 -
1.201 - masm.movptr (boxReg, (int32_t)NULL_WORD) ; // box is really EAX
1.202 - if (os::is_MP()) { masm.lock(); }
1.203 - masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
1.204 - masm.jcc (Assembler::notEqual, LSuccess) ;
1.205 - // Intentional fall-through into slow-path
1.206 -
1.207 - masm.bind (LGoSlowPath) ;
1.208 - masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure
1.209 - masm.jmp (DONE_LABEL) ;
1.210 -
1.211 - masm.bind (LSuccess) ;
1.212 - masm.testl (boxReg, 0) ; // set ICC.ZF=1 to indicate success
1.213 - masm.jmp (DONE_LABEL) ;
1.214 - }
1.215 -
1.216 - masm.bind (Stacked) ;
1.217 - masm.movptr(tmpReg, Address (boxReg, 0)) ; // re-fetch
1.218 - if (os::is_MP()) { masm.lock(); }
1.219 - masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
1.220 -
1.221 - if (EmitSync & 65536) {
1.222 - masm.bind (CheckSucc) ;
1.223 - }
1.224 - masm.bind(DONE_LABEL);
1.225 - if (EmitSync & 32768) {
1.226 - masm.nop(); // avoid branch to branch
1.227 - }
1.228 - }
1.229 - %}
1.230 -
1.231 -
1.232 enc_class enc_rethrow()
1.233 %{
1.234 cbuf.set_insts_mark();
1.235 @@ -11443,27 +11218,25 @@
1.236 // ============================================================================
1.237 // inlined locking and unlocking
1.238
1.239 -instruct cmpFastLock(rFlagsReg cr,
1.240 - rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr)
1.241 -%{
1.242 +instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
1.243 match(Set cr (FastLock object box));
1.244 effect(TEMP tmp, TEMP scr, USE_KILL box);
1.245 -
1.246 ins_cost(300);
1.247 format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
1.248 - ins_encode(Fast_Lock(object, box, tmp, scr));
1.249 + ins_encode %{
1.250 + __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $scr$$Register, _counters);
1.251 + %}
1.252 ins_pipe(pipe_slow);
1.253 %}
1.254
1.255 -instruct cmpFastUnlock(rFlagsReg cr,
1.256 - rRegP object, rax_RegP box, rRegP tmp)
1.257 -%{
1.258 +instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
1.259 match(Set cr (FastUnlock object box));
1.260 effect(TEMP tmp, USE_KILL box);
1.261 -
1.262 ins_cost(300);
1.263 format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
1.264 - ins_encode(Fast_Unlock(object, box, tmp));
1.265 + ins_encode %{
1.266 + __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
1.267 + %}
1.268 ins_pipe(pipe_slow);
1.269 %}
1.270
