1685 |
1686 |
1686 MacroAssembler _masm(&cbuf); |
1687 MacroAssembler _masm(&cbuf); |
1687 // Compare super with sub directly, since super is not in its own SSA. |
1688 // Compare super with sub directly, since super is not in its own SSA. |
1688 // The compiler used to emit this test, but we fold it in here, |
1689 // The compiler used to emit this test, but we fold it in here, |
1689 // to allow platform-specific tweaking on sparc. |
1690 // to allow platform-specific tweaking on sparc. |
1690 __ cmpl(Reax, Resi); |
1691 __ cmpptr(Reax, Resi); |
1691 __ jcc(Assembler::equal, hit); |
1692 __ jcc(Assembler::equal, hit); |
1692 #ifndef PRODUCT |
1693 #ifndef PRODUCT |
1693 __ increment(ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr)); |
1694 __ incrementl(ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr)); |
1694 #endif //PRODUCT |
1695 #endif //PRODUCT |
1695 __ movl(Redi,Address(Resi,sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes())); |
1696 __ movptr(Redi,Address(Resi,sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes())); |
1696 __ movl(Recx,Address(Redi,arrayOopDesc::length_offset_in_bytes())); |
1697 __ movl(Recx,Address(Redi,arrayOopDesc::length_offset_in_bytes())); |
1697 __ addl(Redi,arrayOopDesc::base_offset_in_bytes(T_OBJECT)); |
1698 __ addptr(Redi,arrayOopDesc::base_offset_in_bytes(T_OBJECT)); |
1698 __ repne_scan(); |
1699 __ repne_scan(); |
1699 __ jcc(Assembler::notEqual, miss); |
1700 __ jcc(Assembler::notEqual, miss); |
1700 __ movl(Address(Resi,sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes()),Reax); |
1701 __ movptr(Address(Resi,sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes()),Reax); |
1701 __ bind(hit); |
1702 __ bind(hit); |
1702 if( $primary ) |
1703 if( $primary ) |
1703 __ xorl(Redi,Redi); |
1704 __ xorptr(Redi,Redi); |
1704 __ bind(miss); |
1705 __ bind(miss); |
1705 %} |
1706 %} |
1706 |
1707 |
1707 enc_class FFree_Float_Stack_All %{ // Free_Float_Stack_All |
1708 enc_class FFree_Float_Stack_All %{ // Free_Float_Stack_All |
1708 MacroAssembler masm(&cbuf); |
1709 MacroAssembler masm(&cbuf); |
1747 // mode the result needs to be removed from the FPU stack. It's |
1748 // mode the result needs to be removed from the FPU stack. It's |
1748 // likely that this function call could be removed by the |
1749 // likely that this function call could be removed by the |
1749 // optimizer if the C function is a pure function. |
1750 // optimizer if the C function is a pure function. |
1750 __ ffree(0); |
1751 __ ffree(0); |
1751 } else if (rt == T_FLOAT) { |
1752 } else if (rt == T_FLOAT) { |
1752 __ leal(rsp, Address(rsp, -4)); |
1753 __ lea(rsp, Address(rsp, -4)); |
1753 __ fstp_s(Address(rsp, 0)); |
1754 __ fstp_s(Address(rsp, 0)); |
1754 __ movflt(xmm0, Address(rsp, 0)); |
1755 __ movflt(xmm0, Address(rsp, 0)); |
1755 __ leal(rsp, Address(rsp, 4)); |
1756 __ lea(rsp, Address(rsp, 4)); |
1756 } else if (rt == T_DOUBLE) { |
1757 } else if (rt == T_DOUBLE) { |
1757 __ leal(rsp, Address(rsp, -8)); |
1758 __ lea(rsp, Address(rsp, -8)); |
1758 __ fstp_d(Address(rsp, 0)); |
1759 __ fstp_d(Address(rsp, 0)); |
1759 __ movdbl(xmm0, Address(rsp, 0)); |
1760 __ movdbl(xmm0, Address(rsp, 0)); |
1760 __ leal(rsp, Address(rsp, 8)); |
1761 __ lea(rsp, Address(rsp, 8)); |
1761 } |
1762 } |
1762 } |
1763 } |
1763 %} |
1764 %} |
1764 |
1765 |
1765 |
1766 |
3256 if (_counters != NULL) { |
3257 if (_counters != NULL) { |
3257 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr())); |
3258 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr())); |
3258 } |
3259 } |
3259 if (EmitSync & 1) { |
3260 if (EmitSync & 1) { |
3260 // set box->dhw = unused_mark (3) |
3261 // set box->dhw = unused_mark (3) |
3261 // Force all sync thru slow-path: slow_enter() and slow_exit() |
3262 // Force all sync thru slow-path: slow_enter() and slow_exit() |
3262 masm.movl (Address(boxReg, 0), intptr_t(markOopDesc::unused_mark())) ; |
3263 masm.movptr (Address(boxReg, 0), int32_t(markOopDesc::unused_mark())) ; |
3263 masm.cmpl (rsp, 0) ; |
3264 masm.cmpptr (rsp, (int32_t)0) ; |
3264 } else |
3265 } else |
3265 if (EmitSync & 2) { |
3266 if (EmitSync & 2) { |
3266 Label DONE_LABEL ; |
3267 Label DONE_LABEL ; |
3267 if (UseBiasedLocking) { |
3268 if (UseBiasedLocking) { |
3268 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument. |
3269 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument. |
3269 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); |
3270 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); |
3270 } |
3271 } |
3271 |
3272 |
3272 masm.movl (tmpReg, Address(objReg, 0)) ; // fetch markword |
3273 masm.movptr(tmpReg, Address(objReg, 0)) ; // fetch markword |
3273 masm.orl (tmpReg, 0x1); |
3274 masm.orptr (tmpReg, 0x1); |
3274 masm.movl (Address(boxReg, 0), tmpReg); // Anticipate successful CAS |
3275 masm.movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS |
3275 if (os::is_MP()) { masm.lock(); } |
3276 if (os::is_MP()) { masm.lock(); } |
3276 masm.cmpxchg(boxReg, Address(objReg, 0)); // Updates tmpReg |
3277 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg |
3277 masm.jcc(Assembler::equal, DONE_LABEL); |
3278 masm.jcc(Assembler::equal, DONE_LABEL); |
3278 // Recursive locking |
3279 // Recursive locking |
3279 masm.subl(tmpReg, rsp); |
3280 masm.subptr(tmpReg, rsp); |
3280 masm.andl(tmpReg, 0xFFFFF003 ); |
3281 masm.andptr(tmpReg, (int32_t) 0xFFFFF003 ); |
3281 masm.movl(Address(boxReg, 0), tmpReg); |
3282 masm.movptr(Address(boxReg, 0), tmpReg); |
3282 masm.bind(DONE_LABEL) ; |
3283 masm.bind(DONE_LABEL) ; |
3283 } else { |
3284 } else { |
3284 // Possible cases that we'll encounter in fast_lock |
3285 // Possible cases that we'll encounter in fast_lock |
3285 // ------------------------------------------------ |
3286 // ------------------------------------------------ |
3286 // * Inflated |
3287 // * Inflated |
3287 // -- unlocked |
3288 // -- unlocked |
3288 // -- Locked |
3289 // -- Locked |
3289 // = by self |
3290 // = by self |
3308 // If this invariant is not held we risk exclusion (safety) failure. |
3309 // If this invariant is not held we risk exclusion (safety) failure. |
3309 if (UseBiasedLocking) { |
3310 if (UseBiasedLocking) { |
3310 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); |
3311 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); |
3311 } |
3312 } |
3312 |
3313 |
3313 masm.movl (tmpReg, Address(objReg, 0)) ; // [FETCH] |
3314 masm.movptr(tmpReg, Address(objReg, 0)) ; // [FETCH] |
3314 masm.testl (tmpReg, 0x02) ; // Inflated v (Stack-locked or neutral) |
3315 masm.testptr(tmpReg, 0x02) ; // Inflated v (Stack-locked or neutral) |
3315 masm.jccb (Assembler::notZero, IsInflated) ; |
3316 masm.jccb (Assembler::notZero, IsInflated) ; |
3316 |
3317 |
3317 // Attempt stack-locking ... |
3318 // Attempt stack-locking ... |
3318 masm.orl (tmpReg, 0x1); |
3319 masm.orptr (tmpReg, 0x1); |
3319 masm.movl (Address(boxReg, 0), tmpReg); // Anticipate successful CAS |
3320 masm.movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS |
3320 if (os::is_MP()) { masm.lock(); } |
3321 if (os::is_MP()) { masm.lock(); } |
3321 masm.cmpxchg(boxReg, Address(objReg, 0)); // Updates tmpReg |
3322 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg |
3322 if (_counters != NULL) { |
3323 if (_counters != NULL) { |
3323 masm.cond_inc32(Assembler::equal, |
3324 masm.cond_inc32(Assembler::equal, |
3324 ExternalAddress((address)_counters->fast_path_entry_count_addr())); |
3325 ExternalAddress((address)_counters->fast_path_entry_count_addr())); |
3325 } |
3326 } |
3326 masm.jccb (Assembler::equal, DONE_LABEL); |
3327 masm.jccb (Assembler::equal, DONE_LABEL); |
3327 |
3328 |
3328 // Recursive locking |
3329 // Recursive locking |
3329 masm.subl(tmpReg, rsp); |
3330 masm.subptr(tmpReg, rsp); |
3330 masm.andl(tmpReg, 0xFFFFF003 ); |
3331 masm.andptr(tmpReg, 0xFFFFF003 ); |
3331 masm.movl(Address(boxReg, 0), tmpReg); |
3332 masm.movptr(Address(boxReg, 0), tmpReg); |
3332 if (_counters != NULL) { |
3333 if (_counters != NULL) { |
3333 masm.cond_inc32(Assembler::equal, |
3334 masm.cond_inc32(Assembler::equal, |
3334 ExternalAddress((address)_counters->fast_path_entry_count_addr())); |
3335 ExternalAddress((address)_counters->fast_path_entry_count_addr())); |
3335 } |
3336 } |
3336 masm.jmp (DONE_LABEL) ; |
3337 masm.jmp (DONE_LABEL) ; |
3358 // We'd like to write: |
3359 // We'd like to write: |
3359 // set box->_displaced_header = markOop::unused_mark(). Any non-0 value suffices. |
3360 // set box->_displaced_header = markOop::unused_mark(). Any non-0 value suffices. |
3360 // This is convenient but results a ST-before-CAS penalty. The following CAS suffers |
3361 // This is convenient but results a ST-before-CAS penalty. The following CAS suffers |
3361 // additional latency as we have another ST in the store buffer that must drain. |
3362 // additional latency as we have another ST in the store buffer that must drain. |
3362 |
3363 |
3363 if (EmitSync & 8192) { |
3364 if (EmitSync & 8192) { |
3364 masm.movl (Address(boxReg, 0), 3) ; // results in ST-before-CAS penalty |
3365 masm.movptr(Address(boxReg, 0), 3) ; // results in ST-before-CAS penalty |
3365 masm.get_thread (scrReg) ; |
3366 masm.get_thread (scrReg) ; |
3366 masm.movl (boxReg, tmpReg); // consider: LEA box, [tmp-2] |
3367 masm.movptr(boxReg, tmpReg); // consider: LEA box, [tmp-2] |
3367 masm.movl (tmpReg, 0); // consider: xor vs mov |
3368 masm.movptr(tmpReg, 0); // consider: xor vs mov |
3368 if (os::is_MP()) { masm.lock(); } |
3369 if (os::is_MP()) { masm.lock(); } |
3369 masm.cmpxchg (scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3370 masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3370 } else |
3371 } else |
3371 if ((EmitSync & 128) == 0) { // avoid ST-before-CAS |
3372 if ((EmitSync & 128) == 0) { // avoid ST-before-CAS |
3372 masm.movl (scrReg, boxReg) ; |
3373 masm.movptr(scrReg, boxReg) ; |
3373 masm.movl (boxReg, tmpReg); // consider: LEA box, [tmp-2] |
3374 masm.movptr(boxReg, tmpReg); // consider: LEA box, [tmp-2] |
3374 |
3375 |
3375 // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes |
3376 // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes |
3376 if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) { |
3377 if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) { |
3377 // prefetchw [eax + Offset(_owner)-2] |
3378 // prefetchw [eax + Offset(_owner)-2] |
3378 masm.emit_raw (0x0F) ; |
3379 masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2)); |
3379 masm.emit_raw (0x0D) ; |
|
3380 masm.emit_raw (0x48) ; |
|
3381 masm.emit_raw (ObjectMonitor::owner_offset_in_bytes()-2) ; |
|
3382 } |
3380 } |
3383 |
3381 |
3384 if ((EmitSync & 64) == 0) { |
3382 if ((EmitSync & 64) == 0) { |
3385 // Optimistic form: consider XORL tmpReg,tmpReg |
3383 // Optimistic form: consider XORL tmpReg,tmpReg |
3386 masm.movl (tmpReg, 0 ) ; |
3384 masm.movptr(tmpReg, 0 ) ; |
3387 } else { |
3385 } else { |
3388 // Can suffer RTS->RTO upgrades on shared or cold $ lines |
3386 // Can suffer RTS->RTO upgrades on shared or cold $ lines |
3389 // Test-And-CAS instead of CAS |
3387 // Test-And-CAS instead of CAS |
3390 masm.movl (tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; // rax, = m->_owner |
3388 masm.movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; // rax, = m->_owner |
3391 masm.testl (tmpReg, tmpReg) ; // Locked ? |
3389 masm.testptr(tmpReg, tmpReg) ; // Locked ? |
3392 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3390 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3393 } |
3391 } |
3394 |
3392 |
3395 // Appears unlocked - try to swing _owner from null to non-null. |
3393 // Appears unlocked - try to swing _owner from null to non-null. |
3396 // Ideally, I'd manifest "Self" with get_thread and then attempt |
3394 // Ideally, I'd manifest "Self" with get_thread and then attempt |
3397 // to CAS the register containing Self into m->Owner. |
3395 // to CAS the register containing Self into m->Owner. |
3399 // rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds |
3397 // rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds |
3400 // we later store "Self" into m->Owner. Transiently storing a stack address |
3398 // we later store "Self" into m->Owner. Transiently storing a stack address |
3401 // (rsp or the address of the box) into m->owner is harmless. |
3399 // (rsp or the address of the box) into m->owner is harmless. |
3402 // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
3400 // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
3403 if (os::is_MP()) { masm.lock(); } |
3401 if (os::is_MP()) { masm.lock(); } |
3404 masm.cmpxchg (scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3402 masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3405 masm.movl (Address(scrReg, 0), 3) ; // box->_displaced_header = 3 |
3403 masm.movptr(Address(scrReg, 0), 3) ; // box->_displaced_header = 3 |
3406 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3404 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3407 masm.get_thread (scrReg) ; // beware: clobbers ICCs |
3405 masm.get_thread (scrReg) ; // beware: clobbers ICCs |
3408 masm.movl (Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2), scrReg) ; |
3406 masm.movptr(Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2), scrReg) ; |
3409 masm.xorl (boxReg, boxReg) ; // set icc.ZFlag = 1 to indicate success |
3407 masm.xorptr(boxReg, boxReg) ; // set icc.ZFlag = 1 to indicate success |
3410 |
3408 |
3411 // If the CAS fails we can either retry or pass control to the slow-path. |
3409 // If the CAS fails we can either retry or pass control to the slow-path. |
3412 // We use the latter tactic. |
3410 // We use the latter tactic. |
3413 // Pass the CAS result in the icc.ZFlag into DONE_LABEL |
3411 // Pass the CAS result in the icc.ZFlag into DONE_LABEL |
3414 // If the CAS was successful ... |
3412 // If the CAS was successful ... |
3415 // Self has acquired the lock |
3413 // Self has acquired the lock |
3416 // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. |
3414 // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. |
3417 // Intentional fall-through into DONE_LABEL ... |
3415 // Intentional fall-through into DONE_LABEL ... |
3418 } else { |
3416 } else { |
3419 masm.movl (Address(boxReg, 0), 3) ; // results in ST-before-CAS penalty |
3417 masm.movptr(Address(boxReg, 0), 3) ; // results in ST-before-CAS penalty |
3420 masm.movl (boxReg, tmpReg) ; |
3418 masm.movptr(boxReg, tmpReg) ; |
3421 |
3419 |
3422 // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes |
3420 // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes |
3423 if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) { |
3421 if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) { |
3424 // prefetchw [eax + Offset(_owner)-2] |
3422 // prefetchw [eax + Offset(_owner)-2] |
3425 masm.emit_raw (0x0F) ; |
3423 masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2)); |
3426 masm.emit_raw (0x0D) ; |
|
3427 masm.emit_raw (0x48) ; |
|
3428 masm.emit_raw (ObjectMonitor::owner_offset_in_bytes()-2) ; |
|
3429 } |
3424 } |
3430 |
3425 |
3431 if ((EmitSync & 64) == 0) { |
3426 if ((EmitSync & 64) == 0) { |
3432 // Optimistic form |
3427 // Optimistic form |
3433 masm.xorl (tmpReg, tmpReg) ; |
3428 masm.xorptr (tmpReg, tmpReg) ; |
3434 } else { |
3429 } else { |
3435 // Can suffer RTS->RTO upgrades on shared or cold $ lines |
3430 // Can suffer RTS->RTO upgrades on shared or cold $ lines |
3436 masm.movl (tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; // rax, = m->_owner |
3431 masm.movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; // rax, = m->_owner |
3437 masm.testl (tmpReg, tmpReg) ; // Locked ? |
3432 masm.testptr(tmpReg, tmpReg) ; // Locked ? |
3438 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3433 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3439 } |
3434 } |
3440 |
3435 |
3441 // Appears unlocked - try to swing _owner from null to non-null. |
3436 // Appears unlocked - try to swing _owner from null to non-null. |
3442 // Use either "Self" (in scr) or rsp as thread identity in _owner. |
3437 // Use either "Self" (in scr) or rsp as thread identity in _owner. |
3443 // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
3438 // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
3444 masm.get_thread (scrReg) ; |
3439 masm.get_thread (scrReg) ; |
3445 if (os::is_MP()) { masm.lock(); } |
3440 if (os::is_MP()) { masm.lock(); } |
3446 masm.cmpxchg (scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3441 masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3447 |
3442 |
3448 // If the CAS fails we can either retry or pass control to the slow-path. |
3443 // If the CAS fails we can either retry or pass control to the slow-path. |
3449 // We use the latter tactic. |
3444 // We use the latter tactic. |
3450 // Pass the CAS result in the icc.ZFlag into DONE_LABEL |
3445 // Pass the CAS result in the icc.ZFlag into DONE_LABEL |
3451 // If the CAS was successful ... |
3446 // If the CAS was successful ... |
3512 guarantee (boxReg == as_Register(EAX_enc), "") ; |
3507 guarantee (boxReg == as_Register(EAX_enc), "") ; |
3513 MacroAssembler masm(&cbuf); |
3508 MacroAssembler masm(&cbuf); |
3514 |
3509 |
3515 if (EmitSync & 4) { |
3510 if (EmitSync & 4) { |
3516 // Disable - inhibit all inlining. Force control through the slow-path |
3511 // Disable - inhibit all inlining. Force control through the slow-path |
3517 masm.cmpl (rsp, 0) ; |
3512 masm.cmpptr (rsp, 0) ; |
3518 } else |
3513 } else |
3519 if (EmitSync & 8) { |
3514 if (EmitSync & 8) { |
3520 Label DONE_LABEL ; |
3515 Label DONE_LABEL ; |
3521 if (UseBiasedLocking) { |
3516 if (UseBiasedLocking) { |
3522 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); |
3517 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); |
3523 } |
3518 } |
3524 // classic stack-locking code ... |
3519 // classic stack-locking code ... |
3525 masm.movl (tmpReg, Address(boxReg, 0)) ; |
3520 masm.movptr(tmpReg, Address(boxReg, 0)) ; |
3526 masm.testl (tmpReg, tmpReg) ; |
3521 masm.testptr(tmpReg, tmpReg) ; |
3527 masm.jcc (Assembler::zero, DONE_LABEL) ; |
3522 masm.jcc (Assembler::zero, DONE_LABEL) ; |
3528 if (os::is_MP()) { masm.lock(); } |
3523 if (os::is_MP()) { masm.lock(); } |
3529 masm.cmpxchg(tmpReg, Address(objReg, 0)); // Uses EAX which is box |
3524 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses EAX which is box |
3530 masm.bind(DONE_LABEL); |
3525 masm.bind(DONE_LABEL); |
3531 } else { |
3526 } else { |
3532 Label DONE_LABEL, Stacked, CheckSucc, Inflated ; |
3527 Label DONE_LABEL, Stacked, CheckSucc, Inflated ; |
3533 |
3528 |
3534 // Critically, the biased locking test must have precedence over |
3529 // Critically, the biased locking test must have precedence over |
3535 // and appear before the (box->dhw == 0) recursive stack-lock test. |
3530 // and appear before the (box->dhw == 0) recursive stack-lock test. |
3536 if (UseBiasedLocking) { |
3531 if (UseBiasedLocking) { |
3537 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); |
3532 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); |
3538 } |
3533 } |
3539 |
3534 |
3540 masm.cmpl (Address(boxReg, 0), 0) ; // Examine the displaced header |
3535 masm.cmpptr(Address(boxReg, 0), 0) ; // Examine the displaced header |
3541 masm.movl (tmpReg, Address(objReg, 0)) ; // Examine the object's markword |
3536 masm.movptr(tmpReg, Address(objReg, 0)) ; // Examine the object's markword |
3542 masm.jccb (Assembler::zero, DONE_LABEL) ; // 0 indicates recursive stack-lock |
3537 masm.jccb (Assembler::zero, DONE_LABEL) ; // 0 indicates recursive stack-lock |
3543 |
3538 |
3544 masm.testl (tmpReg, 0x02) ; // Inflated? |
3539 masm.testptr(tmpReg, 0x02) ; // Inflated? |
3545 masm.jccb (Assembler::zero, Stacked) ; |
3540 masm.jccb (Assembler::zero, Stacked) ; |
3546 |
3541 |
3547 masm.bind (Inflated) ; |
3542 masm.bind (Inflated) ; |
3548 // It's inflated. |
3543 // It's inflated. |
3549 // Despite our balanced locking property we still check that m->_owner == Self |
3544 // Despite our balanced locking property we still check that m->_owner == Self |
3569 // each other and there's no need for an explicit barrier (fence). |
3564 // each other and there's no need for an explicit barrier (fence). |
3570 // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html. |
3565 // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html. |
3571 |
3566 |
3572 masm.get_thread (boxReg) ; |
3567 masm.get_thread (boxReg) ; |
3573 if ((EmitSync & 4096) && VM_Version::supports_3dnow() && os::is_MP()) { |
3568 if ((EmitSync & 4096) && VM_Version::supports_3dnow() && os::is_MP()) { |
3574 // prefetchw [ebx + Offset(_owner)-2] |
3569 // prefetchw [ebx + Offset(_owner)-2] |
3575 masm.emit_raw (0x0F) ; |
3570 masm.prefetchw(Address(rbx, ObjectMonitor::owner_offset_in_bytes()-2)); |
3576 masm.emit_raw (0x0D) ; |
|
3577 masm.emit_raw (0x4B) ; |
|
3578 masm.emit_raw (ObjectMonitor::owner_offset_in_bytes()-2) ; |
|
3579 } |
3571 } |
3580 |
3572 |
3581 // Note that we could employ various encoding schemes to reduce |
3573 // Note that we could employ various encoding schemes to reduce |
3582 // the number of loads below (currently 4) to just 2 or 3. |
3574 // the number of loads below (currently 4) to just 2 or 3. |
3583 // Refer to the comments in synchronizer.cpp. |
3575 // Refer to the comments in synchronizer.cpp. |
3584 // In practice the chain of fetches doesn't seem to impact performance, however. |
3576 // In practice the chain of fetches doesn't seem to impact performance, however. |
3585 if ((EmitSync & 65536) == 0 && (EmitSync & 256)) { |
3577 if ((EmitSync & 65536) == 0 && (EmitSync & 256)) { |
3586 // Attempt to reduce branch density - AMD's branch predictor. |
3578 // Attempt to reduce branch density - AMD's branch predictor. |
3587 masm.xorl (boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3579 masm.xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3588 masm.orl (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; |
3580 masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; |
3589 masm.orl (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; |
3581 masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; |
3590 masm.orl (boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; |
3582 masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; |
3591 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3583 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3592 masm.movl (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; |
3584 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; |
3593 masm.jmpb (DONE_LABEL) ; |
3585 masm.jmpb (DONE_LABEL) ; |
3594 } else { |
3586 } else { |
3595 masm.xorl (boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3587 masm.xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3596 masm.orl (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; |
3588 masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; |
3597 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3589 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3598 masm.movl (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; |
3590 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; |
3599 masm.orl (boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; |
3591 masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; |
3600 masm.jccb (Assembler::notZero, CheckSucc) ; |
3592 masm.jccb (Assembler::notZero, CheckSucc) ; |
3601 masm.movl (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; |
3593 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; |
3602 masm.jmpb (DONE_LABEL) ; |
3594 masm.jmpb (DONE_LABEL) ; |
3603 } |
3595 } |
3604 |
3596 |
3605 // The Following code fragment (EmitSync & 65536) improves the performance of |
3597 // The Following code fragment (EmitSync & 65536) improves the performance of |
3606 // contended applications and contended synchronization microbenchmarks. |
3598 // contended applications and contended synchronization microbenchmarks. |
3607 // Unfortunately the emission of the code - even though not executed - causes regressions |
3599 // Unfortunately the emission of the code - even though not executed - causes regressions |
3643 // we just stored into _owner, it's likely that the $line |
3635 // we just stored into _owner, it's likely that the $line |
3644 // remains in M-state for the lock:orl. |
3636 // remains in M-state for the lock:orl. |
3645 // |
3637 // |
3646 // We currently use (3), although it's likely that switching to (2) |
3638 // We currently use (3), although it's likely that switching to (2) |
3647 // is correct for the future. |
3639 // is correct for the future. |
3648 |
3640 |
3649 masm.movl (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; |
3641 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; |
3650 if (os::is_MP()) { |
3642 if (os::is_MP()) { |
3651 if (VM_Version::supports_sse2() && 1 == FenceInstruction) { |
3643 if (VM_Version::supports_sse2() && 1 == FenceInstruction) { |
3652 masm.emit_raw (0x0F) ; // MFENCE ... |
3644 masm.mfence(); |
3653 masm.emit_raw (0xAE) ; |
3645 } else { |
3654 masm.emit_raw (0xF0) ; |
3646 masm.lock () ; masm.addptr(Address(rsp, 0), 0) ; |
3655 } else { |
|
3656 masm.lock () ; masm.addl (Address(rsp, 0), 0) ; |
|
3657 } |
3647 } |
3658 } |
3648 } |
3659 // Ratify _succ remains non-null |
3649 // Ratify _succ remains non-null |
3660 masm.cmpl (Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; |
3650 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; |
3661 masm.jccb (Assembler::notZero, LSuccess) ; |
3651 masm.jccb (Assembler::notZero, LSuccess) ; |
3662 |
3652 |
3663 masm.xorl (boxReg, boxReg) ; // box is really EAX |
3653 masm.xorptr(boxReg, boxReg) ; // box is really EAX |
3664 if (os::is_MP()) { masm.lock(); } |
3654 if (os::is_MP()) { masm.lock(); } |
3665 masm.cmpxchg(rsp, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)); |
3655 masm.cmpxchgptr(rsp, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)); |
3666 masm.jccb (Assembler::notEqual, LSuccess) ; |
3656 masm.jccb (Assembler::notEqual, LSuccess) ; |
3667 // Since we're low on registers we installed rsp as a placeholding in _owner. |
3657 // Since we're low on registers we installed rsp as a placeholding in _owner. |
3668 // Now install Self over rsp. This is safe as we're transitioning from |
3658 // Now install Self over rsp. This is safe as we're transitioning from |
3669 // non-null to non=null |
3659 // non-null to non=null |
3670 masm.get_thread (boxReg) ; |
3660 masm.get_thread (boxReg) ; |
3671 masm.movl (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), boxReg) ; |
3661 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), boxReg) ; |
3672 // Intentional fall-through into LGoSlowPath ... |
3662 // Intentional fall-through into LGoSlowPath ... |
3673 |
3663 |
3674 masm.bind (LGoSlowPath) ; |
3664 masm.bind (LGoSlowPath) ; |
3675 masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure |
3665 masm.orptr(boxReg, 1) ; // set ICC.ZF=0 to indicate failure |
3676 masm.jmpb (DONE_LABEL) ; |
3666 masm.jmpb (DONE_LABEL) ; |
3677 |
3667 |
3678 masm.bind (LSuccess) ; |
3668 masm.bind (LSuccess) ; |
3679 masm.xorl (boxReg, boxReg) ; // set ICC.ZF=1 to indicate success |
3669 masm.xorptr(boxReg, boxReg) ; // set ICC.ZF=1 to indicate success |
3680 masm.jmpb (DONE_LABEL) ; |
3670 masm.jmpb (DONE_LABEL) ; |
3681 } |
3671 } |
3682 |
3672 |
3683 masm.bind (Stacked) ; |
3673 masm.bind (Stacked) ; |
3684 // It's not inflated and it's not recursively stack-locked and it's not biased. |
3674 // It's not inflated and it's not recursively stack-locked and it's not biased. |
3685 // It must be stack-locked. |
3675 // It must be stack-locked. |
3686 // Try to reset the header to displaced header. |
3676 // Try to reset the header to displaced header. |
3687 // The "box" value on the stack is stable, so we can reload |
3677 // The "box" value on the stack is stable, so we can reload |
3688 // and be assured we observe the same value as above. |
3678 // and be assured we observe the same value as above. |
3689 masm.movl (tmpReg, Address(boxReg, 0)) ; |
3679 masm.movptr(tmpReg, Address(boxReg, 0)) ; |
3690 if (os::is_MP()) { masm.lock(); } |
3680 if (os::is_MP()) { masm.lock(); } |
3691 masm.cmpxchg(tmpReg, Address(objReg, 0)); // Uses EAX which is box |
3681 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses EAX which is box |
3692 // Intention fall-thru into DONE_LABEL |
3682 // Intention fall-thru into DONE_LABEL |
3693 |
3683 |
3694 |
3684 |
3695 // DONE_LABEL is a hot target - we'd really like to place it at the |
3685 // DONE_LABEL is a hot target - we'd really like to place it at the |
3696 // start of cache line by padding with NOPs. |
3686 // start of cache line by padding with NOPs. |
3718 int value_offset = java_lang_String::value_offset_in_bytes(); |
3708 int value_offset = java_lang_String::value_offset_in_bytes(); |
3719 int offset_offset = java_lang_String::offset_offset_in_bytes(); |
3709 int offset_offset = java_lang_String::offset_offset_in_bytes(); |
3720 int count_offset = java_lang_String::count_offset_in_bytes(); |
3710 int count_offset = java_lang_String::count_offset_in_bytes(); |
3721 int base_offset = arrayOopDesc::base_offset_in_bytes(T_CHAR); |
3711 int base_offset = arrayOopDesc::base_offset_in_bytes(T_CHAR); |
3722 |
3712 |
3723 masm.movl(rax, Address(rsi, value_offset)); |
3713 masm.movptr(rax, Address(rsi, value_offset)); |
3724 masm.movl(rcx, Address(rsi, offset_offset)); |
3714 masm.movl(rcx, Address(rsi, offset_offset)); |
3725 masm.leal(rax, Address(rax, rcx, Address::times_2, base_offset)); |
3715 masm.lea(rax, Address(rax, rcx, Address::times_2, base_offset)); |
3726 masm.movl(rbx, Address(rdi, value_offset)); |
3716 masm.movptr(rbx, Address(rdi, value_offset)); |
3727 masm.movl(rcx, Address(rdi, offset_offset)); |
3717 masm.movl(rcx, Address(rdi, offset_offset)); |
3728 masm.leal(rbx, Address(rbx, rcx, Address::times_2, base_offset)); |
3718 masm.lea(rbx, Address(rbx, rcx, Address::times_2, base_offset)); |
3729 |
3719 |
3730 // Compute the minimum of the string lengths(rsi) and the |
3720 // Compute the minimum of the string lengths(rsi) and the |
3731 // difference of the string lengths (stack) |
3721 // difference of the string lengths (stack) |
3732 |
3722 |
3733 |
3723 |
3734 if (VM_Version::supports_cmov()) { |
3724 if (VM_Version::supports_cmov()) { |
3735 masm.movl(rdi, Address(rdi, count_offset)); |
3725 masm.movl(rdi, Address(rdi, count_offset)); |
3736 masm.movl(rsi, Address(rsi, count_offset)); |
3726 masm.movl(rsi, Address(rsi, count_offset)); |
3737 masm.movl(rcx, rdi); |
3727 masm.movl(rcx, rdi); |
3738 masm.subl(rdi, rsi); |
3728 masm.subl(rdi, rsi); |
3739 masm.pushl(rdi); |
3729 masm.push(rdi); |
3740 masm.cmovl(Assembler::lessEqual, rsi, rcx); |
3730 masm.cmovl(Assembler::lessEqual, rsi, rcx); |
3741 } else { |
3731 } else { |
3742 masm.movl(rdi, Address(rdi, count_offset)); |
3732 masm.movl(rdi, Address(rdi, count_offset)); |
3743 masm.movl(rcx, Address(rsi, count_offset)); |
3733 masm.movl(rcx, Address(rsi, count_offset)); |
3744 masm.movl(rsi, rdi); |
3734 masm.movl(rsi, rdi); |
3745 masm.subl(rdi, rcx); |
3735 masm.subl(rdi, rcx); |
3746 masm.pushl(rdi); |
3736 masm.push(rdi); |
3747 masm.jcc(Assembler::lessEqual, ECX_GOOD_LABEL); |
3737 masm.jcc(Assembler::lessEqual, ECX_GOOD_LABEL); |
3748 masm.movl(rsi, rcx); |
3738 masm.movl(rsi, rcx); |
3749 // rsi holds min, rcx is unused |
3739 // rsi holds min, rcx is unused |
3750 } |
3740 } |
3751 |
3741 |
3759 masm.load_unsigned_word(rdi, Address(rax, 0)); |
3749 masm.load_unsigned_word(rdi, Address(rax, 0)); |
3760 |
3750 |
3761 // Compare first characters |
3751 // Compare first characters |
3762 masm.subl(rcx, rdi); |
3752 masm.subl(rcx, rdi); |
3763 masm.jcc(Assembler::notZero, POP_LABEL); |
3753 masm.jcc(Assembler::notZero, POP_LABEL); |
3764 masm.decrement(rsi); |
3754 masm.decrementl(rsi); |
3765 masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
3755 masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
3766 |
3756 |
3767 { |
3757 { |
3768 // Check after comparing first character to see if strings are equivalent |
3758 // Check after comparing first character to see if strings are equivalent |
3769 Label LSkip2; |
3759 Label LSkip2; |
3770 // Check if the strings start at same location |
3760 // Check if the strings start at same location |
3771 masm.cmpl(rbx,rax); |
3761 masm.cmpptr(rbx,rax); |
3772 masm.jcc(Assembler::notEqual, LSkip2); |
3762 masm.jcc(Assembler::notEqual, LSkip2); |
3773 |
3763 |
3774 // Check if the length difference is zero (from stack) |
3764 // Check if the length difference is zero (from stack) |
3775 masm.cmpl(Address(rsp, 0), 0x0); |
3765 masm.cmpl(Address(rsp, 0), 0x0); |
3776 masm.jcc(Assembler::equal, LENGTH_DIFF_LABEL); |
3766 masm.jcc(Assembler::equal, LENGTH_DIFF_LABEL); |
3778 // Strings might not be equivalent |
3768 // Strings might not be equivalent |
3779 masm.bind(LSkip2); |
3769 masm.bind(LSkip2); |
3780 } |
3770 } |
3781 |
3771 |
3782 // Shift rax, and rbx, to the end of the arrays, negate min |
3772 // Shift rax, and rbx, to the end of the arrays, negate min |
3783 masm.leal(rax, Address(rax, rsi, Address::times_2, 2)); |
3773 masm.lea(rax, Address(rax, rsi, Address::times_2, 2)); |
3784 masm.leal(rbx, Address(rbx, rsi, Address::times_2, 2)); |
3774 masm.lea(rbx, Address(rbx, rsi, Address::times_2, 2)); |
3785 masm.negl(rsi); |
3775 masm.negl(rsi); |
3786 |
3776 |
3787 // Compare the rest of the characters |
3777 // Compare the rest of the characters |
3788 masm.bind(WHILE_HEAD_LABEL); |
3778 masm.bind(WHILE_HEAD_LABEL); |
3789 masm.load_unsigned_word(rcx, Address(rbx, rsi, Address::times_2, 0)); |
3779 masm.load_unsigned_word(rcx, Address(rbx, rsi, Address::times_2, 0)); |
3790 masm.load_unsigned_word(rdi, Address(rax, rsi, Address::times_2, 0)); |
3780 masm.load_unsigned_word(rdi, Address(rax, rsi, Address::times_2, 0)); |
3791 masm.subl(rcx, rdi); |
3781 masm.subl(rcx, rdi); |
3792 masm.jcc(Assembler::notZero, POP_LABEL); |
3782 masm.jcc(Assembler::notZero, POP_LABEL); |
3793 masm.increment(rsi); |
3783 masm.incrementl(rsi); |
3794 masm.jcc(Assembler::notZero, WHILE_HEAD_LABEL); |
3784 masm.jcc(Assembler::notZero, WHILE_HEAD_LABEL); |
3795 |
3785 |
3796 // Strings are equal up to min length. Return the length difference. |
3786 // Strings are equal up to min length. Return the length difference. |
3797 masm.bind(LENGTH_DIFF_LABEL); |
3787 masm.bind(LENGTH_DIFF_LABEL); |
3798 masm.popl(rcx); |
3788 masm.pop(rcx); |
3799 masm.jmp(DONE_LABEL); |
3789 masm.jmp(DONE_LABEL); |
3800 |
3790 |
3801 // Discard the stored length difference |
3791 // Discard the stored length difference |
3802 masm.bind(POP_LABEL); |
3792 masm.bind(POP_LABEL); |
3803 masm.addl(rsp, 4); |
3793 masm.addptr(rsp, 4); |
3804 |
3794 |
3805 // That's it |
3795 // That's it |
3806 masm.bind(DONE_LABEL); |
3796 masm.bind(DONE_LABEL); |
3807 %} |
3797 %} |
3808 |
3798 |
3809 enc_class enc_Array_Equals(eDIRegP ary1, eSIRegP ary2, eAXRegI tmp1, eBXRegI tmp2, eCXRegI result) %{ |
3799 enc_class enc_Array_Equals(eDIRegP ary1, eSIRegP ary2, eAXRegI tmp1, eBXRegI tmp2, eCXRegI result) %{ |
12260 "JMP,s done\n" |
12251 "JMP,s done\n" |
12261 "m_one:\tDEC $dst\n" |
12252 "m_one:\tDEC $dst\n" |
12262 "done:" %} |
12253 "done:" %} |
12263 ins_encode %{ |
12254 ins_encode %{ |
12264 Label p_one, m_one, done; |
12255 Label p_one, m_one, done; |
12265 __ xorl($dst$$Register, $dst$$Register); |
12256 __ xorptr($dst$$Register, $dst$$Register); |
12266 __ cmpl(HIGH_FROM_LOW($src1$$Register), HIGH_FROM_LOW($src2$$Register)); |
12257 __ cmpl(HIGH_FROM_LOW($src1$$Register), HIGH_FROM_LOW($src2$$Register)); |
12267 __ jccb(Assembler::less, m_one); |
12258 __ jccb(Assembler::less, m_one); |
12268 __ jccb(Assembler::greater, p_one); |
12259 __ jccb(Assembler::greater, p_one); |
12269 __ cmpl($src1$$Register, $src2$$Register); |
12260 __ cmpl($src1$$Register, $src2$$Register); |
12270 __ jccb(Assembler::below, m_one); |
12261 __ jccb(Assembler::below, m_one); |
12271 __ jccb(Assembler::equal, done); |
12262 __ jccb(Assembler::equal, done); |
12272 __ bind(p_one); |
12263 __ bind(p_one); |
12273 __ increment($dst$$Register); |
12264 __ incrementl($dst$$Register); |
12274 __ jmpb(done); |
12265 __ jmpb(done); |
12275 __ bind(m_one); |
12266 __ bind(m_one); |
12276 __ decrement($dst$$Register); |
12267 __ decrementl($dst$$Register); |
12277 __ bind(done); |
12268 __ bind(done); |
12278 %} |
12269 %} |
12279 ins_pipe( pipe_slow ); |
12270 ins_pipe( pipe_slow ); |
12280 %} |
12271 %} |
12281 |
12272 |