aoqi@0: /* aoqi@0: * Copyright (c) 1998, 2014, Oracle and/or its affiliates. All rights reserved. aoqi@0: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. aoqi@0: * aoqi@0: * This code is free software; you can redistribute it and/or modify it aoqi@0: * under the terms of the GNU General Public License version 2 only, as aoqi@0: * published by the Free Software Foundation. aoqi@0: * aoqi@0: * This code is distributed in the hope that it will be useful, but WITHOUT aoqi@0: * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or aoqi@0: * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License aoqi@0: * version 2 for more details (a copy is included in the LICENSE file that aoqi@0: * accompanied this code). aoqi@0: * aoqi@0: * You should have received a copy of the GNU General Public License version aoqi@0: * 2 along with this work; if not, write to the Free Software Foundation, aoqi@0: * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. aoqi@0: * aoqi@0: * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA aoqi@0: * or visit www.oracle.com if you need additional information or have any aoqi@0: * questions. aoqi@0: * aoqi@0: */ aoqi@0: aoqi@0: #include "precompiled.hpp" aoqi@0: #include "classfile/vmSymbols.hpp" aoqi@0: #include "memory/resourceArea.hpp" aoqi@0: #include "oops/markOop.hpp" aoqi@0: #include "oops/oop.inline.hpp" aoqi@0: #include "runtime/biasedLocking.hpp" aoqi@0: #include "runtime/handles.inline.hpp" aoqi@0: #include "runtime/interfaceSupport.hpp" aoqi@0: #include "runtime/mutexLocker.hpp" aoqi@0: #include "runtime/objectMonitor.hpp" aoqi@0: #include "runtime/objectMonitor.inline.hpp" aoqi@0: #include "runtime/osThread.hpp" aoqi@0: #include "runtime/stubRoutines.hpp" aoqi@0: #include "runtime/synchronizer.hpp" aoqi@0: #include "runtime/thread.inline.hpp" aoqi@0: #include "utilities/dtrace.hpp" aoqi@0: #include "utilities/events.hpp" aoqi@0: #include "utilities/preserveException.hpp" aoqi@0: #ifdef TARGET_OS_FAMILY_linux aoqi@0: # include "os_linux.inline.hpp" aoqi@0: #endif aoqi@0: #ifdef TARGET_OS_FAMILY_solaris aoqi@0: # include "os_solaris.inline.hpp" aoqi@0: #endif aoqi@0: #ifdef TARGET_OS_FAMILY_windows aoqi@0: # include "os_windows.inline.hpp" aoqi@0: #endif aoqi@0: #ifdef TARGET_OS_FAMILY_bsd aoqi@0: # include "os_bsd.inline.hpp" aoqi@0: #endif aoqi@0: aoqi@0: #if defined(__GNUC__) && !defined(PPC64) aoqi@0: // Need to inhibit inlining for older versions of GCC to avoid build-time failures aoqi@0: #define ATTR __attribute__((noinline)) aoqi@0: #else aoqi@0: #define ATTR aoqi@0: #endif aoqi@0: aoqi@0: PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC aoqi@0: aoqi@0: // The "core" versions of monitor enter and exit reside in this file. aoqi@0: // The interpreter and compilers contain specialized transliterated aoqi@0: // variants of the enter-exit fast-path operations. See i486.ad fast_lock(), aoqi@0: // for instance. If you make changes here, make sure to modify the aoqi@0: // interpreter, and both C1 and C2 fast-path inline locking code emission. aoqi@0: // aoqi@0: // aoqi@0: // ----------------------------------------------------------------------------- aoqi@0: aoqi@0: #ifdef DTRACE_ENABLED aoqi@0: aoqi@0: // Only bother with this argument setup if dtrace is available aoqi@0: // TODO-FIXME: probes should not fire when caller is _blocked. assert() accordingly. aoqi@0: aoqi@0: #define DTRACE_MONITOR_PROBE_COMMON(obj, thread) \ aoqi@0: char* bytes = NULL; \ aoqi@0: int len = 0; \ aoqi@0: jlong jtid = SharedRuntime::get_java_tid(thread); \ aoqi@0: Symbol* klassname = ((oop)(obj))->klass()->name(); \ aoqi@0: if (klassname != NULL) { \ aoqi@0: bytes = (char*)klassname->bytes(); \ aoqi@0: len = klassname->utf8_length(); \ aoqi@0: } aoqi@0: aoqi@0: #ifndef USDT2 aoqi@0: HS_DTRACE_PROBE_DECL5(hotspot, monitor__wait, aoqi@0: jlong, uintptr_t, char*, int, long); aoqi@0: HS_DTRACE_PROBE_DECL4(hotspot, monitor__waited, aoqi@0: jlong, uintptr_t, char*, int); aoqi@0: aoqi@0: #define DTRACE_MONITOR_WAIT_PROBE(monitor, obj, thread, millis) \ aoqi@0: { \ aoqi@0: if (DTraceMonitorProbes) { \ aoqi@0: DTRACE_MONITOR_PROBE_COMMON(obj, thread); \ aoqi@0: HS_DTRACE_PROBE5(hotspot, monitor__wait, jtid, \ aoqi@0: (monitor), bytes, len, (millis)); \ aoqi@0: } \ aoqi@0: } aoqi@0: aoqi@0: #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread) \ aoqi@0: { \ aoqi@0: if (DTraceMonitorProbes) { \ aoqi@0: DTRACE_MONITOR_PROBE_COMMON(obj, thread); \ aoqi@0: HS_DTRACE_PROBE4(hotspot, monitor__##probe, jtid, \ aoqi@0: (uintptr_t)(monitor), bytes, len); \ aoqi@0: } \ aoqi@0: } aoqi@0: aoqi@0: #else /* USDT2 */ aoqi@0: aoqi@0: #define DTRACE_MONITOR_WAIT_PROBE(monitor, obj, thread, millis) \ aoqi@0: { \ aoqi@0: if (DTraceMonitorProbes) { \ aoqi@0: DTRACE_MONITOR_PROBE_COMMON(obj, thread); \ aoqi@0: HOTSPOT_MONITOR_WAIT(jtid, \ aoqi@0: (uintptr_t)(monitor), bytes, len, (millis)); \ aoqi@0: } \ aoqi@0: } aoqi@0: aoqi@0: #define HOTSPOT_MONITOR_PROBE_waited HOTSPOT_MONITOR_PROBE_WAITED aoqi@0: aoqi@0: #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread) \ aoqi@0: { \ aoqi@0: if (DTraceMonitorProbes) { \ aoqi@0: DTRACE_MONITOR_PROBE_COMMON(obj, thread); \ aoqi@0: HOTSPOT_MONITOR_PROBE_##probe(jtid, /* probe = waited */ \ aoqi@0: (uintptr_t)(monitor), bytes, len); \ aoqi@0: } \ aoqi@0: } aoqi@0: aoqi@0: #endif /* USDT2 */ aoqi@0: #else // ndef DTRACE_ENABLED aoqi@0: aoqi@0: #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon) {;} aoqi@0: #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon) {;} aoqi@0: aoqi@0: #endif // ndef DTRACE_ENABLED aoqi@0: aoqi@0: // This exists only as a workaround of dtrace bug 6254741 aoqi@0: int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) { aoqi@0: DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr); aoqi@0: return 0; aoqi@0: } aoqi@0: aoqi@0: #define NINFLATIONLOCKS 256 aoqi@0: static volatile intptr_t InflationLocks [NINFLATIONLOCKS] ; aoqi@0: aoqi@0: ObjectMonitor * ObjectSynchronizer::gBlockList = NULL ; aoqi@0: ObjectMonitor * volatile ObjectSynchronizer::gFreeList = NULL ; aoqi@0: ObjectMonitor * volatile ObjectSynchronizer::gOmInUseList = NULL ; aoqi@0: int ObjectSynchronizer::gOmInUseCount = 0; aoqi@0: static volatile intptr_t ListLock = 0 ; // protects global monitor free-list cache aoqi@0: static volatile int MonitorFreeCount = 0 ; // # on gFreeList aoqi@0: static volatile int MonitorPopulation = 0 ; // # Extant -- in circulation aoqi@0: #define CHAINMARKER (cast_to_oop(-1)) aoqi@0: aoqi@0: // ----------------------------------------------------------------------------- aoqi@0: // Fast Monitor Enter/Exit aoqi@0: // This the fast monitor enter. The interpreter and compiler use aoqi@0: // some assembly copies of this code. Make sure update those code aoqi@0: // if the following function is changed. The implementation is aoqi@0: // extremely sensitive to race condition. Be careful. aoqi@0: aoqi@0: void ObjectSynchronizer::fast_enter(Handle obj, BasicLock* lock, bool attempt_rebias, TRAPS) { aoqi@0: if (UseBiasedLocking) { aoqi@0: if (!SafepointSynchronize::is_at_safepoint()) { aoqi@0: BiasedLocking::Condition cond = BiasedLocking::revoke_and_rebias(obj, attempt_rebias, THREAD); aoqi@0: if (cond == BiasedLocking::BIAS_REVOKED_AND_REBIASED) { aoqi@0: return; aoqi@0: } aoqi@0: } else { aoqi@0: assert(!attempt_rebias, "can not rebias toward VM thread"); aoqi@0: BiasedLocking::revoke_at_safepoint(obj); aoqi@0: } aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: aoqi@0: slow_enter (obj, lock, THREAD) ; aoqi@0: } aoqi@0: aoqi@0: void ObjectSynchronizer::fast_exit(oop object, BasicLock* lock, TRAPS) { aoqi@0: assert(!object->mark()->has_bias_pattern(), "should not see bias pattern here"); aoqi@0: // if displaced header is null, the previous enter is recursive enter, no-op aoqi@0: markOop dhw = lock->displaced_header(); aoqi@0: markOop mark ; aoqi@0: if (dhw == NULL) { aoqi@0: // Recursive stack-lock. aoqi@0: // Diagnostics -- Could be: stack-locked, inflating, inflated. aoqi@0: mark = object->mark() ; aoqi@0: assert (!mark->is_neutral(), "invariant") ; aoqi@0: if (mark->has_locker() && mark != markOopDesc::INFLATING()) { aoqi@0: assert(THREAD->is_lock_owned((address)mark->locker()), "invariant") ; aoqi@0: } aoqi@0: if (mark->has_monitor()) { aoqi@0: ObjectMonitor * m = mark->monitor() ; aoqi@0: assert(((oop)(m->object()))->mark() == mark, "invariant") ; aoqi@0: assert(m->is_entered(THREAD), "invariant") ; aoqi@0: } aoqi@0: return ; aoqi@0: } aoqi@0: aoqi@0: mark = object->mark() ; aoqi@0: aoqi@0: // If the object is stack-locked by the current thread, try to aoqi@0: // swing the displaced header from the box back to the mark. aoqi@0: if (mark == (markOop) lock) { aoqi@0: assert (dhw->is_neutral(), "invariant") ; aoqi@0: if ((markOop) Atomic::cmpxchg_ptr (dhw, object->mark_addr(), mark) == mark) { aoqi@0: TEVENT (fast_exit: release stacklock) ; aoqi@0: return; aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: ObjectSynchronizer::inflate(THREAD, object)->exit (true, THREAD) ; aoqi@0: } aoqi@0: aoqi@0: // ----------------------------------------------------------------------------- aoqi@0: // Interpreter/Compiler Slow Case aoqi@0: // This routine is used to handle interpreter/compiler slow case aoqi@0: // We don't need to use fast path here, because it must have been aoqi@0: // failed in the interpreter/compiler code. aoqi@0: void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) { aoqi@0: markOop mark = obj->mark(); aoqi@0: assert(!mark->has_bias_pattern(), "should not see bias pattern here"); aoqi@0: aoqi@0: if (mark->is_neutral()) { aoqi@0: // Anticipate successful CAS -- the ST of the displaced mark must aoqi@0: // be visible <= the ST performed by the CAS. aoqi@0: lock->set_displaced_header(mark); aoqi@0: if (mark == (markOop) Atomic::cmpxchg_ptr(lock, obj()->mark_addr(), mark)) { aoqi@0: TEVENT (slow_enter: release stacklock) ; aoqi@0: return ; aoqi@0: } aoqi@0: // Fall through to inflate() ... aoqi@0: } else aoqi@0: if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) { aoqi@0: assert(lock != mark->locker(), "must not re-lock the same lock"); aoqi@0: assert(lock != (BasicLock*)obj->mark(), "don't relock with same BasicLock"); aoqi@0: lock->set_displaced_header(NULL); aoqi@0: return; aoqi@0: } aoqi@0: aoqi@0: #if 0 aoqi@0: // The following optimization isn't particularly useful. aoqi@0: if (mark->has_monitor() && mark->monitor()->is_entered(THREAD)) { aoqi@0: lock->set_displaced_header (NULL) ; aoqi@0: return ; aoqi@0: } aoqi@0: #endif aoqi@0: aoqi@0: // The object header will never be displaced to this lock, aoqi@0: // so it does not matter what the value is, except that it aoqi@0: // must be non-zero to avoid looking like a re-entrant lock, aoqi@0: // and must not look locked either. aoqi@0: lock->set_displaced_header(markOopDesc::unused_mark()); aoqi@0: ObjectSynchronizer::inflate(THREAD, obj())->enter(THREAD); aoqi@0: } aoqi@0: aoqi@0: // This routine is used to handle interpreter/compiler slow case aoqi@0: // We don't need to use fast path here, because it must have aoqi@0: // failed in the interpreter/compiler code. Simply use the heavy aoqi@0: // weight monitor should be ok, unless someone find otherwise. aoqi@0: void ObjectSynchronizer::slow_exit(oop object, BasicLock* lock, TRAPS) { aoqi@0: fast_exit (object, lock, THREAD) ; aoqi@0: } aoqi@0: aoqi@0: // ----------------------------------------------------------------------------- aoqi@0: // Class Loader support to workaround deadlocks on the class loader lock objects aoqi@0: // Also used by GC aoqi@0: // complete_exit()/reenter() are used to wait on a nested lock aoqi@0: // i.e. to give up an outer lock completely and then re-enter aoqi@0: // Used when holding nested locks - lock acquisition order: lock1 then lock2 aoqi@0: // 1) complete_exit lock1 - saving recursion count aoqi@0: // 2) wait on lock2 aoqi@0: // 3) when notified on lock2, unlock lock2 aoqi@0: // 4) reenter lock1 with original recursion count aoqi@0: // 5) lock lock2 aoqi@0: // NOTE: must use heavy weight monitor to handle complete_exit/reenter() aoqi@0: intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) { aoqi@0: TEVENT (complete_exit) ; aoqi@0: if (UseBiasedLocking) { aoqi@0: BiasedLocking::revoke_and_rebias(obj, false, THREAD); aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: aoqi@0: ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj()); aoqi@0: aoqi@0: return monitor->complete_exit(THREAD); aoqi@0: } aoqi@0: aoqi@0: // NOTE: must use heavy weight monitor to handle complete_exit/reenter() aoqi@0: void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) { aoqi@0: TEVENT (reenter) ; aoqi@0: if (UseBiasedLocking) { aoqi@0: BiasedLocking::revoke_and_rebias(obj, false, THREAD); aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: aoqi@0: ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj()); aoqi@0: aoqi@0: monitor->reenter(recursion, THREAD); aoqi@0: } aoqi@0: // ----------------------------------------------------------------------------- aoqi@0: // JNI locks on java objects aoqi@0: // NOTE: must use heavy weight monitor to handle jni monitor enter aoqi@0: void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) { // possible entry from jni enter aoqi@0: // the current locking is from JNI instead of Java code aoqi@0: TEVENT (jni_enter) ; aoqi@0: if (UseBiasedLocking) { aoqi@0: BiasedLocking::revoke_and_rebias(obj, false, THREAD); aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: THREAD->set_current_pending_monitor_is_from_java(false); aoqi@0: ObjectSynchronizer::inflate(THREAD, obj())->enter(THREAD); aoqi@0: THREAD->set_current_pending_monitor_is_from_java(true); aoqi@0: } aoqi@0: aoqi@0: // NOTE: must use heavy weight monitor to handle jni monitor enter aoqi@0: bool ObjectSynchronizer::jni_try_enter(Handle obj, Thread* THREAD) { aoqi@0: if (UseBiasedLocking) { aoqi@0: BiasedLocking::revoke_and_rebias(obj, false, THREAD); aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: aoqi@0: ObjectMonitor* monitor = ObjectSynchronizer::inflate_helper(obj()); aoqi@0: return monitor->try_enter(THREAD); aoqi@0: } aoqi@0: aoqi@0: aoqi@0: // NOTE: must use heavy weight monitor to handle jni monitor exit aoqi@0: void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) { aoqi@0: TEVENT (jni_exit) ; aoqi@0: if (UseBiasedLocking) { aoqi@0: Handle h_obj(THREAD, obj); aoqi@0: BiasedLocking::revoke_and_rebias(h_obj, false, THREAD); aoqi@0: obj = h_obj(); aoqi@0: } aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: aoqi@0: ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj); aoqi@0: // If this thread has locked the object, exit the monitor. Note: can't use aoqi@0: // monitor->check(CHECK); must exit even if an exception is pending. aoqi@0: if (monitor->check(THREAD)) { aoqi@0: monitor->exit(true, THREAD); aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: // ----------------------------------------------------------------------------- aoqi@0: // Internal VM locks on java objects aoqi@0: // standard constructor, allows locking failures aoqi@0: ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool doLock) { aoqi@0: _dolock = doLock; aoqi@0: _thread = thread; aoqi@0: debug_only(if (StrictSafepointChecks) _thread->check_for_valid_safepoint_state(false);) aoqi@0: _obj = obj; aoqi@0: aoqi@0: if (_dolock) { aoqi@0: TEVENT (ObjectLocker) ; aoqi@0: aoqi@0: ObjectSynchronizer::fast_enter(_obj, &_lock, false, _thread); aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: ObjectLocker::~ObjectLocker() { aoqi@0: if (_dolock) { aoqi@0: ObjectSynchronizer::fast_exit(_obj(), &_lock, _thread); aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: aoqi@0: // ----------------------------------------------------------------------------- aoqi@0: // Wait/Notify/NotifyAll aoqi@0: // NOTE: must use heavy weight monitor to handle wait() aoqi@0: void ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) { aoqi@0: if (UseBiasedLocking) { aoqi@0: BiasedLocking::revoke_and_rebias(obj, false, THREAD); aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: if (millis < 0) { aoqi@0: TEVENT (wait - throw IAX) ; aoqi@0: THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative"); aoqi@0: } aoqi@0: ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj()); aoqi@0: DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis); aoqi@0: monitor->wait(millis, true, THREAD); aoqi@0: aoqi@0: /* This dummy call is in place to get around dtrace bug 6254741. Once aoqi@0: that's fixed we can uncomment the following line and remove the call */ aoqi@0: // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD); aoqi@0: dtrace_waited_probe(monitor, obj, THREAD); aoqi@0: } aoqi@0: aoqi@0: void ObjectSynchronizer::waitUninterruptibly (Handle obj, jlong millis, TRAPS) { aoqi@0: if (UseBiasedLocking) { aoqi@0: BiasedLocking::revoke_and_rebias(obj, false, THREAD); aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: if (millis < 0) { aoqi@0: TEVENT (wait - throw IAX) ; aoqi@0: THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative"); aoqi@0: } aoqi@0: ObjectSynchronizer::inflate(THREAD, obj()) -> wait(millis, false, THREAD) ; aoqi@0: } aoqi@0: aoqi@0: void ObjectSynchronizer::notify(Handle obj, TRAPS) { aoqi@0: if (UseBiasedLocking) { aoqi@0: BiasedLocking::revoke_and_rebias(obj, false, THREAD); aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: aoqi@0: markOop mark = obj->mark(); aoqi@0: if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) { aoqi@0: return; aoqi@0: } aoqi@0: ObjectSynchronizer::inflate(THREAD, obj())->notify(THREAD); aoqi@0: } aoqi@0: aoqi@0: // NOTE: see comment of notify() aoqi@0: void ObjectSynchronizer::notifyall(Handle obj, TRAPS) { aoqi@0: if (UseBiasedLocking) { aoqi@0: BiasedLocking::revoke_and_rebias(obj, false, THREAD); aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: aoqi@0: markOop mark = obj->mark(); aoqi@0: if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) { aoqi@0: return; aoqi@0: } aoqi@0: ObjectSynchronizer::inflate(THREAD, obj())->notifyAll(THREAD); aoqi@0: } aoqi@0: aoqi@0: // ----------------------------------------------------------------------------- aoqi@0: // Hash Code handling aoqi@0: // aoqi@0: // Performance concern: aoqi@0: // OrderAccess::storestore() calls release() which STs 0 into the global volatile aoqi@0: // OrderAccess::Dummy variable. This store is unnecessary for correctness. aoqi@0: // Many threads STing into a common location causes considerable cache migration aoqi@0: // or "sloshing" on large SMP system. As such, I avoid using OrderAccess::storestore() aoqi@0: // until it's repaired. In some cases OrderAccess::fence() -- which incurs local aoqi@0: // latency on the executing processor -- is a better choice as it scales on SMP aoqi@0: // systems. See http://blogs.sun.com/dave/entry/biased_locking_in_hotspot for a aoqi@0: // discussion of coherency costs. Note that all our current reference platforms aoqi@0: // provide strong ST-ST order, so the issue is moot on IA32, x64, and SPARC. aoqi@0: // aoqi@0: // As a general policy we use "volatile" to control compiler-based reordering aoqi@0: // and explicit fences (barriers) to control for architectural reordering performed aoqi@0: // by the CPU(s) or platform. aoqi@0: aoqi@0: struct SharedGlobals { aoqi@0: // These are highly shared mostly-read variables. aoqi@0: // To avoid false-sharing they need to be the sole occupants of a $ line. aoqi@0: double padPrefix [8]; aoqi@0: volatile int stwRandom ; aoqi@0: volatile int stwCycle ; aoqi@0: aoqi@0: // Hot RW variables -- Sequester to avoid false-sharing aoqi@0: double padSuffix [16]; aoqi@0: volatile int hcSequence ; aoqi@0: double padFinal [8] ; aoqi@0: } ; aoqi@0: aoqi@0: static SharedGlobals GVars ; aoqi@0: static int MonitorScavengeThreshold = 1000000 ; aoqi@0: static volatile int ForceMonitorScavenge = 0 ; // Scavenge required and pending aoqi@0: aoqi@0: static markOop ReadStableMark (oop obj) { aoqi@0: markOop mark = obj->mark() ; aoqi@0: if (!mark->is_being_inflated()) { aoqi@0: return mark ; // normal fast-path return aoqi@0: } aoqi@0: aoqi@0: int its = 0 ; aoqi@0: for (;;) { aoqi@0: markOop mark = obj->mark() ; aoqi@0: if (!mark->is_being_inflated()) { aoqi@0: return mark ; // normal fast-path return aoqi@0: } aoqi@0: aoqi@0: // The object is being inflated by some other thread. aoqi@0: // The caller of ReadStableMark() must wait for inflation to complete. aoqi@0: // Avoid live-lock aoqi@0: // TODO: consider calling SafepointSynchronize::do_call_back() while aoqi@0: // spinning to see if there's a safepoint pending. If so, immediately aoqi@0: // yielding or blocking would be appropriate. Avoid spinning while aoqi@0: // there is a safepoint pending. aoqi@0: // TODO: add inflation contention performance counters. aoqi@0: // TODO: restrict the aggregate number of spinners. aoqi@0: aoqi@0: ++its ; aoqi@0: if (its > 10000 || !os::is_MP()) { aoqi@0: if (its & 1) { aoqi@0: os::NakedYield() ; aoqi@0: TEVENT (Inflate: INFLATING - yield) ; aoqi@0: } else { aoqi@0: // Note that the following code attenuates the livelock problem but is not aoqi@0: // a complete remedy. A more complete solution would require that the inflating aoqi@0: // thread hold the associated inflation lock. The following code simply restricts aoqi@0: // the number of spinners to at most one. We'll have N-2 threads blocked aoqi@0: // on the inflationlock, 1 thread holding the inflation lock and using aoqi@0: // a yield/park strategy, and 1 thread in the midst of inflation. aoqi@0: // A more refined approach would be to change the encoding of INFLATING aoqi@0: // to allow encapsulation of a native thread pointer. Threads waiting for aoqi@0: // inflation to complete would use CAS to push themselves onto a singly linked aoqi@0: // list rooted at the markword. Once enqueued, they'd loop, checking a per-thread flag aoqi@0: // and calling park(). When inflation was complete the thread that accomplished inflation aoqi@0: // would detach the list and set the markword to inflated with a single CAS and aoqi@0: // then for each thread on the list, set the flag and unpark() the thread. aoqi@0: // This is conceptually similar to muxAcquire-muxRelease, except that muxRelease aoqi@0: // wakes at most one thread whereas we need to wake the entire list. aoqi@0: int ix = (cast_from_oop(obj) >> 5) & (NINFLATIONLOCKS-1) ; aoqi@0: int YieldThenBlock = 0 ; aoqi@0: assert (ix >= 0 && ix < NINFLATIONLOCKS, "invariant") ; aoqi@0: assert ((NINFLATIONLOCKS & (NINFLATIONLOCKS-1)) == 0, "invariant") ; aoqi@0: Thread::muxAcquire (InflationLocks + ix, "InflationLock") ; aoqi@0: while (obj->mark() == markOopDesc::INFLATING()) { aoqi@0: // Beware: NakedYield() is advisory and has almost no effect on some platforms aoqi@0: // so we periodically call Self->_ParkEvent->park(1). aoqi@0: // We use a mixed spin/yield/block mechanism. aoqi@0: if ((YieldThenBlock++) >= 16) { aoqi@0: Thread::current()->_ParkEvent->park(1) ; aoqi@0: } else { aoqi@0: os::NakedYield() ; aoqi@0: } aoqi@0: } aoqi@0: Thread::muxRelease (InflationLocks + ix ) ; aoqi@0: TEVENT (Inflate: INFLATING - yield/park) ; aoqi@0: } aoqi@0: } else { aoqi@0: SpinPause() ; // SMP-polite spinning aoqi@0: } aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: // hashCode() generation : aoqi@0: // aoqi@0: // Possibilities: aoqi@0: // * MD5Digest of {obj,stwRandom} aoqi@0: // * CRC32 of {obj,stwRandom} or any linear-feedback shift register function. aoqi@0: // * A DES- or AES-style SBox[] mechanism aoqi@0: // * One of the Phi-based schemes, such as: aoqi@0: // 2654435761 = 2^32 * Phi (golden ratio) aoqi@0: // HashCodeValue = ((uintptr_t(obj) >> 3) * 2654435761) ^ GVars.stwRandom ; aoqi@0: // * A variation of Marsaglia's shift-xor RNG scheme. aoqi@0: // * (obj ^ stwRandom) is appealing, but can result aoqi@0: // in undesirable regularity in the hashCode values of adjacent objects aoqi@0: // (objects allocated back-to-back, in particular). This could potentially aoqi@0: // result in hashtable collisions and reduced hashtable efficiency. aoqi@0: // There are simple ways to "diffuse" the middle address bits over the aoqi@0: // generated hashCode values: aoqi@0: // aoqi@0: aoqi@0: static inline intptr_t get_next_hash(Thread * Self, oop obj) { aoqi@0: intptr_t value = 0 ; aoqi@0: if (hashCode == 0) { aoqi@0: // This form uses an unguarded global Park-Miller RNG, aoqi@0: // so it's possible for two threads to race and generate the same RNG. aoqi@0: // On MP system we'll have lots of RW access to a global, so the aoqi@0: // mechanism induces lots of coherency traffic. aoqi@0: value = os::random() ; aoqi@0: } else aoqi@0: if (hashCode == 1) { aoqi@0: // This variation has the property of being stable (idempotent) aoqi@0: // between STW operations. This can be useful in some of the 1-0 aoqi@0: // synchronization schemes. aoqi@0: intptr_t addrBits = cast_from_oop(obj) >> 3 ; aoqi@0: value = addrBits ^ (addrBits >> 5) ^ GVars.stwRandom ; aoqi@0: } else aoqi@0: if (hashCode == 2) { aoqi@0: value = 1 ; // for sensitivity testing aoqi@0: } else aoqi@0: if (hashCode == 3) { aoqi@0: value = ++GVars.hcSequence ; aoqi@0: } else aoqi@0: if (hashCode == 4) { aoqi@0: value = cast_from_oop(obj) ; aoqi@0: } else { aoqi@0: // Marsaglia's xor-shift scheme with thread-specific state aoqi@0: // This is probably the best overall implementation -- we'll aoqi@0: // likely make this the default in future releases. aoqi@0: unsigned t = Self->_hashStateX ; aoqi@0: t ^= (t << 11) ; aoqi@0: Self->_hashStateX = Self->_hashStateY ; aoqi@0: Self->_hashStateY = Self->_hashStateZ ; aoqi@0: Self->_hashStateZ = Self->_hashStateW ; aoqi@0: unsigned v = Self->_hashStateW ; aoqi@0: v = (v ^ (v >> 19)) ^ (t ^ (t >> 8)) ; aoqi@0: Self->_hashStateW = v ; aoqi@0: value = v ; aoqi@0: } aoqi@0: aoqi@0: value &= markOopDesc::hash_mask; aoqi@0: if (value == 0) value = 0xBAD ; aoqi@0: assert (value != markOopDesc::no_hash, "invariant") ; aoqi@0: TEVENT (hashCode: GENERATE) ; aoqi@0: return value; aoqi@0: } aoqi@0: // aoqi@0: intptr_t ObjectSynchronizer::FastHashCode (Thread * Self, oop obj) { aoqi@0: if (UseBiasedLocking) { aoqi@0: // NOTE: many places throughout the JVM do not expect a safepoint aoqi@0: // to be taken here, in particular most operations on perm gen aoqi@0: // objects. However, we only ever bias Java instances and all of aoqi@0: // the call sites of identity_hash that might revoke biases have aoqi@0: // been checked to make sure they can handle a safepoint. The aoqi@0: // added check of the bias pattern is to avoid useless calls to aoqi@0: // thread-local storage. aoqi@0: if (obj->mark()->has_bias_pattern()) { aoqi@0: // Box and unbox the raw reference just in case we cause a STW safepoint. aoqi@0: Handle hobj (Self, obj) ; aoqi@0: // Relaxing assertion for bug 6320749. aoqi@0: assert (Universe::verify_in_progress() || aoqi@0: !SafepointSynchronize::is_at_safepoint(), aoqi@0: "biases should not be seen by VM thread here"); aoqi@0: BiasedLocking::revoke_and_rebias(hobj, false, JavaThread::current()); aoqi@0: obj = hobj() ; aoqi@0: assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: // hashCode() is a heap mutator ... aoqi@0: // Relaxing assertion for bug 6320749. aoqi@0: assert (Universe::verify_in_progress() || aoqi@0: !SafepointSynchronize::is_at_safepoint(), "invariant") ; aoqi@0: assert (Universe::verify_in_progress() || aoqi@0: Self->is_Java_thread() , "invariant") ; aoqi@0: assert (Universe::verify_in_progress() || aoqi@0: ((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant") ; aoqi@0: aoqi@0: ObjectMonitor* monitor = NULL; aoqi@0: markOop temp, test; aoqi@0: intptr_t hash; aoqi@0: markOop mark = ReadStableMark (obj); aoqi@0: aoqi@0: // object should remain ineligible for biased locking aoqi@0: assert (!mark->has_bias_pattern(), "invariant") ; aoqi@0: aoqi@0: if (mark->is_neutral()) { aoqi@0: hash = mark->hash(); // this is a normal header aoqi@0: if (hash) { // if it has hash, just return it aoqi@0: return hash; aoqi@0: } aoqi@0: hash = get_next_hash(Self, obj); // allocate a new hash code aoqi@0: temp = mark->copy_set_hash(hash); // merge the hash code into header aoqi@0: // use (machine word version) atomic operation to install the hash aoqi@0: test = (markOop) Atomic::cmpxchg_ptr(temp, obj->mark_addr(), mark); aoqi@0: if (test == mark) { aoqi@0: return hash; aoqi@0: } aoqi@0: // If atomic operation failed, we must inflate the header aoqi@0: // into heavy weight monitor. We could add more code here aoqi@0: // for fast path, but it does not worth the complexity. aoqi@0: } else if (mark->has_monitor()) { aoqi@0: monitor = mark->monitor(); aoqi@0: temp = monitor->header(); aoqi@0: assert (temp->is_neutral(), "invariant") ; aoqi@0: hash = temp->hash(); aoqi@0: if (hash) { aoqi@0: return hash; aoqi@0: } aoqi@0: // Skip to the following code to reduce code size aoqi@0: } else if (Self->is_lock_owned((address)mark->locker())) { aoqi@0: temp = mark->displaced_mark_helper(); // this is a lightweight monitor owned aoqi@0: assert (temp->is_neutral(), "invariant") ; aoqi@0: hash = temp->hash(); // by current thread, check if the displaced aoqi@0: if (hash) { // header contains hash code aoqi@0: return hash; aoqi@0: } aoqi@0: // WARNING: aoqi@0: // The displaced header is strictly immutable. aoqi@0: // It can NOT be changed in ANY cases. So we have aoqi@0: // to inflate the header into heavyweight monitor aoqi@0: // even the current thread owns the lock. The reason aoqi@0: // is the BasicLock (stack slot) will be asynchronously aoqi@0: // read by other threads during the inflate() function. aoqi@0: // Any change to stack may not propagate to other threads aoqi@0: // correctly. aoqi@0: } aoqi@0: aoqi@0: // Inflate the monitor to set hash code aoqi@0: monitor = ObjectSynchronizer::inflate(Self, obj); aoqi@0: // Load displaced header and check it has hash code aoqi@0: mark = monitor->header(); aoqi@0: assert (mark->is_neutral(), "invariant") ; aoqi@0: hash = mark->hash(); aoqi@0: if (hash == 0) { aoqi@0: hash = get_next_hash(Self, obj); aoqi@0: temp = mark->copy_set_hash(hash); // merge hash code into header aoqi@0: assert (temp->is_neutral(), "invariant") ; aoqi@0: test = (markOop) Atomic::cmpxchg_ptr(temp, monitor, mark); aoqi@0: if (test != mark) { aoqi@0: // The only update to the header in the monitor (outside GC) aoqi@0: // is install the hash code. If someone add new usage of aoqi@0: // displaced header, please update this code aoqi@0: hash = test->hash(); aoqi@0: assert (test->is_neutral(), "invariant") ; aoqi@0: assert (hash != 0, "Trivial unexpected object/monitor header usage."); aoqi@0: } aoqi@0: } aoqi@0: // We finally get the hash aoqi@0: return hash; aoqi@0: } aoqi@0: aoqi@0: // Deprecated -- use FastHashCode() instead. aoqi@0: aoqi@0: intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) { aoqi@0: return FastHashCode (Thread::current(), obj()) ; aoqi@0: } aoqi@0: aoqi@0: aoqi@0: bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread, aoqi@0: Handle h_obj) { aoqi@0: if (UseBiasedLocking) { aoqi@0: BiasedLocking::revoke_and_rebias(h_obj, false, thread); aoqi@0: assert(!h_obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: aoqi@0: assert(thread == JavaThread::current(), "Can only be called on current thread"); aoqi@0: oop obj = h_obj(); aoqi@0: aoqi@0: markOop mark = ReadStableMark (obj) ; aoqi@0: aoqi@0: // Uncontended case, header points to stack aoqi@0: if (mark->has_locker()) { aoqi@0: return thread->is_lock_owned((address)mark->locker()); aoqi@0: } aoqi@0: // Contended case, header points to ObjectMonitor (tagged pointer) aoqi@0: if (mark->has_monitor()) { aoqi@0: ObjectMonitor* monitor = mark->monitor(); aoqi@0: return monitor->is_entered(thread) != 0 ; aoqi@0: } aoqi@0: // Unlocked case, header in place aoqi@0: assert(mark->is_neutral(), "sanity check"); aoqi@0: return false; aoqi@0: } aoqi@0: aoqi@0: // Be aware of this method could revoke bias of the lock object. aoqi@0: // This method querys the ownership of the lock handle specified by 'h_obj'. aoqi@0: // If the current thread owns the lock, it returns owner_self. If no aoqi@0: // thread owns the lock, it returns owner_none. Otherwise, it will return aoqi@0: // ower_other. aoqi@0: ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership aoqi@0: (JavaThread *self, Handle h_obj) { aoqi@0: // The caller must beware this method can revoke bias, and aoqi@0: // revocation can result in a safepoint. aoqi@0: assert (!SafepointSynchronize::is_at_safepoint(), "invariant") ; aoqi@0: assert (self->thread_state() != _thread_blocked , "invariant") ; aoqi@0: aoqi@0: // Possible mark states: neutral, biased, stack-locked, inflated aoqi@0: aoqi@0: if (UseBiasedLocking && h_obj()->mark()->has_bias_pattern()) { aoqi@0: // CASE: biased aoqi@0: BiasedLocking::revoke_and_rebias(h_obj, false, self); aoqi@0: assert(!h_obj->mark()->has_bias_pattern(), aoqi@0: "biases should be revoked by now"); aoqi@0: } aoqi@0: aoqi@0: assert(self == JavaThread::current(), "Can only be called on current thread"); aoqi@0: oop obj = h_obj(); aoqi@0: markOop mark = ReadStableMark (obj) ; aoqi@0: aoqi@0: // CASE: stack-locked. Mark points to a BasicLock on the owner's stack. aoqi@0: if (mark->has_locker()) { aoqi@0: return self->is_lock_owned((address)mark->locker()) ? aoqi@0: owner_self : owner_other; aoqi@0: } aoqi@0: aoqi@0: // CASE: inflated. Mark (tagged pointer) points to an objectMonitor. aoqi@0: // The Object:ObjectMonitor relationship is stable as long as we're aoqi@0: // not at a safepoint. aoqi@0: if (mark->has_monitor()) { aoqi@0: void * owner = mark->monitor()->_owner ; aoqi@0: if (owner == NULL) return owner_none ; aoqi@0: return (owner == self || aoqi@0: self->is_lock_owned((address)owner)) ? owner_self : owner_other; aoqi@0: } aoqi@0: aoqi@0: // CASE: neutral aoqi@0: assert(mark->is_neutral(), "sanity check"); aoqi@0: return owner_none ; // it's unlocked aoqi@0: } aoqi@0: aoqi@0: // FIXME: jvmti should call this aoqi@0: JavaThread* ObjectSynchronizer::get_lock_owner(Handle h_obj, bool doLock) { aoqi@0: if (UseBiasedLocking) { aoqi@0: if (SafepointSynchronize::is_at_safepoint()) { aoqi@0: BiasedLocking::revoke_at_safepoint(h_obj); aoqi@0: } else { aoqi@0: BiasedLocking::revoke_and_rebias(h_obj, false, JavaThread::current()); aoqi@0: } aoqi@0: assert(!h_obj->mark()->has_bias_pattern(), "biases should be revoked by now"); aoqi@0: } aoqi@0: aoqi@0: oop obj = h_obj(); aoqi@0: address owner = NULL; aoqi@0: aoqi@0: markOop mark = ReadStableMark (obj) ; aoqi@0: aoqi@0: // Uncontended case, header points to stack aoqi@0: if (mark->has_locker()) { aoqi@0: owner = (address) mark->locker(); aoqi@0: } aoqi@0: aoqi@0: // Contended case, header points to ObjectMonitor (tagged pointer) aoqi@0: if (mark->has_monitor()) { aoqi@0: ObjectMonitor* monitor = mark->monitor(); aoqi@0: assert(monitor != NULL, "monitor should be non-null"); aoqi@0: owner = (address) monitor->owner(); aoqi@0: } aoqi@0: aoqi@0: if (owner != NULL) { aoqi@0: // owning_thread_from_monitor_owner() may also return NULL here aoqi@0: return Threads::owning_thread_from_monitor_owner(owner, doLock); aoqi@0: } aoqi@0: aoqi@0: // Unlocked case, header in place aoqi@0: // Cannot have assertion since this object may have been aoqi@0: // locked by another thread when reaching here. aoqi@0: // assert(mark->is_neutral(), "sanity check"); aoqi@0: aoqi@0: return NULL; aoqi@0: } aoqi@0: // Visitors ... aoqi@0: aoqi@0: void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) { aoqi@0: ObjectMonitor* block = gBlockList; aoqi@0: ObjectMonitor* mid; aoqi@0: while (block) { aoqi@0: assert(block->object() == CHAINMARKER, "must be a block header"); aoqi@0: for (int i = _BLOCKSIZE - 1; i > 0; i--) { aoqi@0: mid = block + i; aoqi@0: oop object = (oop) mid->object(); aoqi@0: if (object != NULL) { aoqi@0: closure->do_monitor(mid); aoqi@0: } aoqi@0: } aoqi@0: block = (ObjectMonitor*) block->FreeNext; aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: // Get the next block in the block list. aoqi@0: static inline ObjectMonitor* next(ObjectMonitor* block) { aoqi@0: assert(block->object() == CHAINMARKER, "must be a block header"); aoqi@0: block = block->FreeNext ; aoqi@0: assert(block == NULL || block->object() == CHAINMARKER, "must be a block header"); aoqi@0: return block; aoqi@0: } aoqi@0: aoqi@0: aoqi@0: void ObjectSynchronizer::oops_do(OopClosure* f) { aoqi@0: assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); aoqi@0: for (ObjectMonitor* block = gBlockList; block != NULL; block = next(block)) { aoqi@0: assert(block->object() == CHAINMARKER, "must be a block header"); aoqi@0: for (int i = 1; i < _BLOCKSIZE; i++) { aoqi@0: ObjectMonitor* mid = &block[i]; aoqi@0: if (mid->object() != NULL) { aoqi@0: f->do_oop((oop*)mid->object_addr()); aoqi@0: } aoqi@0: } aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: aoqi@0: // ----------------------------------------------------------------------------- aoqi@0: // ObjectMonitor Lifecycle aoqi@0: // ----------------------- aoqi@0: // Inflation unlinks monitors from the global gFreeList and aoqi@0: // associates them with objects. Deflation -- which occurs at aoqi@0: // STW-time -- disassociates idle monitors from objects. Such aoqi@0: // scavenged monitors are returned to the gFreeList. aoqi@0: // aoqi@0: // The global list is protected by ListLock. All the critical sections aoqi@0: // are short and operate in constant-time. aoqi@0: // aoqi@0: // ObjectMonitors reside in type-stable memory (TSM) and are immortal. aoqi@0: // aoqi@0: // Lifecycle: aoqi@0: // -- unassigned and on the global free list aoqi@0: // -- unassigned and on a thread's private omFreeList aoqi@0: // -- assigned to an object. The object is inflated and the mark refers aoqi@0: // to the objectmonitor. aoqi@0: // aoqi@0: aoqi@0: aoqi@0: // Constraining monitor pool growth via MonitorBound ... aoqi@0: // aoqi@0: // The monitor pool is grow-only. We scavenge at STW safepoint-time, but the aoqi@0: // the rate of scavenging is driven primarily by GC. As such, we can find aoqi@0: // an inordinate number of monitors in circulation. aoqi@0: // To avoid that scenario we can artificially induce a STW safepoint aoqi@0: // if the pool appears to be growing past some reasonable bound. aoqi@0: // Generally we favor time in space-time tradeoffs, but as there's no aoqi@0: // natural back-pressure on the # of extant monitors we need to impose some aoqi@0: // type of limit. Beware that if MonitorBound is set to too low a value aoqi@0: // we could just loop. In addition, if MonitorBound is set to a low value aoqi@0: // we'll incur more safepoints, which are harmful to performance. aoqi@0: // See also: GuaranteedSafepointInterval aoqi@0: // aoqi@0: // The current implementation uses asynchronous VM operations. aoqi@0: // aoqi@0: aoqi@0: static void InduceScavenge (Thread * Self, const char * Whence) { aoqi@0: // Induce STW safepoint to trim monitors aoqi@0: // Ultimately, this results in a call to deflate_idle_monitors() in the near future. aoqi@0: // More precisely, trigger an asynchronous STW safepoint as the number aoqi@0: // of active monitors passes the specified threshold. aoqi@0: // TODO: assert thread state is reasonable aoqi@0: aoqi@0: if (ForceMonitorScavenge == 0 && Atomic::xchg (1, &ForceMonitorScavenge) == 0) { aoqi@0: if (ObjectMonitor::Knob_Verbose) { aoqi@0: ::printf ("Monitor scavenge - Induced STW @%s (%d)\n", Whence, ForceMonitorScavenge) ; aoqi@0: ::fflush(stdout) ; aoqi@0: } aoqi@0: // Induce a 'null' safepoint to scavenge monitors aoqi@0: // Must VM_Operation instance be heap allocated as the op will be enqueue and posted aoqi@0: // to the VMthread and have a lifespan longer than that of this activation record. aoqi@0: // The VMThread will delete the op when completed. aoqi@0: VMThread::execute (new VM_ForceAsyncSafepoint()) ; aoqi@0: aoqi@0: if (ObjectMonitor::Knob_Verbose) { aoqi@0: ::printf ("Monitor scavenge - STW posted @%s (%d)\n", Whence, ForceMonitorScavenge) ; aoqi@0: ::fflush(stdout) ; aoqi@0: } aoqi@0: } aoqi@0: } aoqi@0: /* Too slow for general assert or debug aoqi@0: void ObjectSynchronizer::verifyInUse (Thread *Self) { aoqi@0: ObjectMonitor* mid; aoqi@0: int inusetally = 0; aoqi@0: for (mid = Self->omInUseList; mid != NULL; mid = mid->FreeNext) { aoqi@0: inusetally ++; aoqi@0: } aoqi@0: assert(inusetally == Self->omInUseCount, "inuse count off"); aoqi@0: aoqi@0: int freetally = 0; aoqi@0: for (mid = Self->omFreeList; mid != NULL; mid = mid->FreeNext) { aoqi@0: freetally ++; aoqi@0: } aoqi@0: assert(freetally == Self->omFreeCount, "free count off"); aoqi@0: } aoqi@0: */ aoqi@0: ObjectMonitor * ATTR ObjectSynchronizer::omAlloc (Thread * Self) { aoqi@0: // A large MAXPRIVATE value reduces both list lock contention aoqi@0: // and list coherency traffic, but also tends to increase the aoqi@0: // number of objectMonitors in circulation as well as the STW aoqi@0: // scavenge costs. As usual, we lean toward time in space-time aoqi@0: // tradeoffs. aoqi@0: const int MAXPRIVATE = 1024 ; aoqi@0: for (;;) { aoqi@0: ObjectMonitor * m ; aoqi@0: aoqi@0: // 1: try to allocate from the thread's local omFreeList. aoqi@0: // Threads will attempt to allocate first from their local list, then aoqi@0: // from the global list, and only after those attempts fail will the thread aoqi@0: // attempt to instantiate new monitors. Thread-local free lists take aoqi@0: // heat off the ListLock and improve allocation latency, as well as reducing aoqi@0: // coherency traffic on the shared global list. aoqi@0: m = Self->omFreeList ; aoqi@0: if (m != NULL) { aoqi@0: Self->omFreeList = m->FreeNext ; aoqi@0: Self->omFreeCount -- ; aoqi@0: // CONSIDER: set m->FreeNext = BAD -- diagnostic hygiene aoqi@0: guarantee (m->object() == NULL, "invariant") ; aoqi@0: if (MonitorInUseLists) { aoqi@0: m->FreeNext = Self->omInUseList; aoqi@0: Self->omInUseList = m; aoqi@0: Self->omInUseCount ++; aoqi@0: // verifyInUse(Self); aoqi@0: } else { aoqi@0: m->FreeNext = NULL; aoqi@0: } aoqi@0: return m ; aoqi@0: } aoqi@0: aoqi@0: // 2: try to allocate from the global gFreeList aoqi@0: // CONSIDER: use muxTry() instead of muxAcquire(). aoqi@0: // If the muxTry() fails then drop immediately into case 3. aoqi@0: // If we're using thread-local free lists then try aoqi@0: // to reprovision the caller's free list. aoqi@0: if (gFreeList != NULL) { aoqi@0: // Reprovision the thread's omFreeList. aoqi@0: // Use bulk transfers to reduce the allocation rate and heat aoqi@0: // on various locks. aoqi@0: Thread::muxAcquire (&ListLock, "omAlloc") ; aoqi@0: for (int i = Self->omFreeProvision; --i >= 0 && gFreeList != NULL; ) { aoqi@0: MonitorFreeCount --; aoqi@0: ObjectMonitor * take = gFreeList ; aoqi@0: gFreeList = take->FreeNext ; aoqi@0: guarantee (take->object() == NULL, "invariant") ; aoqi@0: guarantee (!take->is_busy(), "invariant") ; aoqi@0: take->Recycle() ; aoqi@0: omRelease (Self, take, false) ; aoqi@0: } aoqi@0: Thread::muxRelease (&ListLock) ; aoqi@0: Self->omFreeProvision += 1 + (Self->omFreeProvision/2) ; aoqi@0: if (Self->omFreeProvision > MAXPRIVATE ) Self->omFreeProvision = MAXPRIVATE ; aoqi@0: TEVENT (omFirst - reprovision) ; aoqi@0: aoqi@0: const int mx = MonitorBound ; aoqi@0: if (mx > 0 && (MonitorPopulation-MonitorFreeCount) > mx) { aoqi@0: // We can't safely induce a STW safepoint from omAlloc() as our thread aoqi@0: // state may not be appropriate for such activities and callers may hold aoqi@0: // naked oops, so instead we defer the action. aoqi@0: InduceScavenge (Self, "omAlloc") ; aoqi@0: } aoqi@0: continue; aoqi@0: } aoqi@0: aoqi@0: // 3: allocate a block of new ObjectMonitors aoqi@0: // Both the local and global free lists are empty -- resort to malloc(). aoqi@0: // In the current implementation objectMonitors are TSM - immortal. aoqi@0: assert (_BLOCKSIZE > 1, "invariant") ; aoqi@0: ObjectMonitor * temp = new ObjectMonitor[_BLOCKSIZE]; aoqi@0: aoqi@0: // NOTE: (almost) no way to recover if allocation failed. aoqi@0: // We might be able to induce a STW safepoint and scavenge enough aoqi@0: // objectMonitors to permit progress. aoqi@0: if (temp == NULL) { aoqi@0: vm_exit_out_of_memory (sizeof (ObjectMonitor[_BLOCKSIZE]), OOM_MALLOC_ERROR, aoqi@0: "Allocate ObjectMonitors"); aoqi@0: } aoqi@0: aoqi@0: // Format the block. aoqi@0: // initialize the linked list, each monitor points to its next aoqi@0: // forming the single linked free list, the very first monitor aoqi@0: // will points to next block, which forms the block list. aoqi@0: // The trick of using the 1st element in the block as gBlockList aoqi@0: // linkage should be reconsidered. A better implementation would aoqi@0: // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; } aoqi@0: aoqi@0: for (int i = 1; i < _BLOCKSIZE ; i++) { aoqi@0: temp[i].FreeNext = &temp[i+1]; aoqi@0: } aoqi@0: aoqi@0: // terminate the last monitor as the end of list aoqi@0: temp[_BLOCKSIZE - 1].FreeNext = NULL ; aoqi@0: aoqi@0: // Element [0] is reserved for global list linkage aoqi@0: temp[0].set_object(CHAINMARKER); aoqi@0: aoqi@0: // Consider carving out this thread's current request from the aoqi@0: // block in hand. This avoids some lock traffic and redundant aoqi@0: // list activity. aoqi@0: aoqi@0: // Acquire the ListLock to manipulate BlockList and FreeList. aoqi@0: // An Oyama-Taura-Yonezawa scheme might be more efficient. aoqi@0: Thread::muxAcquire (&ListLock, "omAlloc [2]") ; aoqi@0: MonitorPopulation += _BLOCKSIZE-1; aoqi@0: MonitorFreeCount += _BLOCKSIZE-1; aoqi@0: aoqi@0: // Add the new block to the list of extant blocks (gBlockList). aoqi@0: // The very first objectMonitor in a block is reserved and dedicated. aoqi@0: // It serves as blocklist "next" linkage. aoqi@0: temp[0].FreeNext = gBlockList; aoqi@0: gBlockList = temp; aoqi@0: aoqi@0: // Add the new string of objectMonitors to the global free list aoqi@0: temp[_BLOCKSIZE - 1].FreeNext = gFreeList ; aoqi@0: gFreeList = temp + 1; aoqi@0: Thread::muxRelease (&ListLock) ; aoqi@0: TEVENT (Allocate block of monitors) ; aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: // Place "m" on the caller's private per-thread omFreeList. aoqi@0: // In practice there's no need to clamp or limit the number of aoqi@0: // monitors on a thread's omFreeList as the only time we'll call aoqi@0: // omRelease is to return a monitor to the free list after a CAS aoqi@0: // attempt failed. This doesn't allow unbounded #s of monitors to aoqi@0: // accumulate on a thread's free list. aoqi@0: // aoqi@0: aoqi@0: void ObjectSynchronizer::omRelease (Thread * Self, ObjectMonitor * m, bool fromPerThreadAlloc) { aoqi@0: guarantee (m->object() == NULL, "invariant") ; aoqi@0: aoqi@0: // Remove from omInUseList aoqi@0: if (MonitorInUseLists && fromPerThreadAlloc) { aoqi@0: ObjectMonitor* curmidinuse = NULL; aoqi@0: for (ObjectMonitor* mid = Self->omInUseList; mid != NULL; ) { aoqi@0: if (m == mid) { aoqi@0: // extract from per-thread in-use-list aoqi@0: if (mid == Self->omInUseList) { aoqi@0: Self->omInUseList = mid->FreeNext; aoqi@0: } else if (curmidinuse != NULL) { aoqi@0: curmidinuse->FreeNext = mid->FreeNext; // maintain the current thread inuselist aoqi@0: } aoqi@0: Self->omInUseCount --; aoqi@0: // verifyInUse(Self); aoqi@0: break; aoqi@0: } else { aoqi@0: curmidinuse = mid; aoqi@0: mid = mid->FreeNext; aoqi@0: } aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: // FreeNext is used for both onInUseList and omFreeList, so clear old before setting new aoqi@0: m->FreeNext = Self->omFreeList ; aoqi@0: Self->omFreeList = m ; aoqi@0: Self->omFreeCount ++ ; aoqi@0: } aoqi@0: aoqi@0: // Return the monitors of a moribund thread's local free list to aoqi@0: // the global free list. Typically a thread calls omFlush() when aoqi@0: // it's dying. We could also consider having the VM thread steal aoqi@0: // monitors from threads that have not run java code over a few aoqi@0: // consecutive STW safepoints. Relatedly, we might decay aoqi@0: // omFreeProvision at STW safepoints. aoqi@0: // aoqi@0: // Also return the monitors of a moribund thread"s omInUseList to aoqi@0: // a global gOmInUseList under the global list lock so these aoqi@0: // will continue to be scanned. aoqi@0: // aoqi@0: // We currently call omFlush() from the Thread:: dtor _after the thread aoqi@0: // has been excised from the thread list and is no longer a mutator. aoqi@0: // That means that omFlush() can run concurrently with a safepoint and aoqi@0: // the scavenge operator. Calling omFlush() from JavaThread::exit() might aoqi@0: // be a better choice as we could safely reason that that the JVM is aoqi@0: // not at a safepoint at the time of the call, and thus there could aoqi@0: // be not inopportune interleavings between omFlush() and the scavenge aoqi@0: // operator. aoqi@0: aoqi@0: void ObjectSynchronizer::omFlush (Thread * Self) { aoqi@0: ObjectMonitor * List = Self->omFreeList ; // Null-terminated SLL aoqi@0: Self->omFreeList = NULL ; aoqi@0: ObjectMonitor * Tail = NULL ; aoqi@0: int Tally = 0; aoqi@0: if (List != NULL) { aoqi@0: ObjectMonitor * s ; aoqi@0: for (s = List ; s != NULL ; s = s->FreeNext) { aoqi@0: Tally ++ ; aoqi@0: Tail = s ; aoqi@0: guarantee (s->object() == NULL, "invariant") ; aoqi@0: guarantee (!s->is_busy(), "invariant") ; aoqi@0: s->set_owner (NULL) ; // redundant but good hygiene aoqi@0: TEVENT (omFlush - Move one) ; aoqi@0: } aoqi@0: guarantee (Tail != NULL && List != NULL, "invariant") ; aoqi@0: } aoqi@0: aoqi@0: ObjectMonitor * InUseList = Self->omInUseList; aoqi@0: ObjectMonitor * InUseTail = NULL ; aoqi@0: int InUseTally = 0; aoqi@0: if (InUseList != NULL) { aoqi@0: Self->omInUseList = NULL; aoqi@0: ObjectMonitor *curom; aoqi@0: for (curom = InUseList; curom != NULL; curom = curom->FreeNext) { aoqi@0: InUseTail = curom; aoqi@0: InUseTally++; aoqi@0: } aoqi@0: // TODO debug aoqi@0: assert(Self->omInUseCount == InUseTally, "inuse count off"); aoqi@0: Self->omInUseCount = 0; aoqi@0: guarantee (InUseTail != NULL && InUseList != NULL, "invariant"); aoqi@0: } aoqi@0: aoqi@0: Thread::muxAcquire (&ListLock, "omFlush") ; aoqi@0: if (Tail != NULL) { aoqi@0: Tail->FreeNext = gFreeList ; aoqi@0: gFreeList = List ; aoqi@0: MonitorFreeCount += Tally; aoqi@0: } aoqi@0: aoqi@0: if (InUseTail != NULL) { aoqi@0: InUseTail->FreeNext = gOmInUseList; aoqi@0: gOmInUseList = InUseList; aoqi@0: gOmInUseCount += InUseTally; aoqi@0: } aoqi@0: aoqi@0: Thread::muxRelease (&ListLock) ; aoqi@0: TEVENT (omFlush) ; aoqi@0: } aoqi@0: aoqi@0: // Fast path code shared by multiple functions aoqi@0: ObjectMonitor* ObjectSynchronizer::inflate_helper(oop obj) { aoqi@0: markOop mark = obj->mark(); aoqi@0: if (mark->has_monitor()) { aoqi@0: assert(ObjectSynchronizer::verify_objmon_isinpool(mark->monitor()), "monitor is invalid"); aoqi@0: assert(mark->monitor()->header()->is_neutral(), "monitor must record a good object header"); aoqi@0: return mark->monitor(); aoqi@0: } aoqi@0: return ObjectSynchronizer::inflate(Thread::current(), obj); aoqi@0: } aoqi@0: aoqi@0: aoqi@0: // Note that we could encounter some performance loss through false-sharing as aoqi@0: // multiple locks occupy the same $ line. Padding might be appropriate. aoqi@0: aoqi@0: aoqi@0: ObjectMonitor * ATTR ObjectSynchronizer::inflate (Thread * Self, oop object) { aoqi@0: // Inflate mutates the heap ... aoqi@0: // Relaxing assertion for bug 6320749. aoqi@0: assert (Universe::verify_in_progress() || aoqi@0: !SafepointSynchronize::is_at_safepoint(), "invariant") ; aoqi@0: aoqi@0: for (;;) { aoqi@0: const markOop mark = object->mark() ; aoqi@0: assert (!mark->has_bias_pattern(), "invariant") ; aoqi@0: aoqi@0: // The mark can be in one of the following states: aoqi@0: // * Inflated - just return aoqi@0: // * Stack-locked - coerce it to inflated aoqi@0: // * INFLATING - busy wait for conversion to complete aoqi@0: // * Neutral - aggressively inflate the object. aoqi@0: // * BIASED - Illegal. We should never see this aoqi@0: aoqi@0: // CASE: inflated aoqi@0: if (mark->has_monitor()) { aoqi@0: ObjectMonitor * inf = mark->monitor() ; aoqi@0: assert (inf->header()->is_neutral(), "invariant"); aoqi@0: assert (inf->object() == object, "invariant") ; aoqi@0: assert (ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid"); aoqi@0: return inf ; aoqi@0: } aoqi@0: aoqi@0: // CASE: inflation in progress - inflating over a stack-lock. aoqi@0: // Some other thread is converting from stack-locked to inflated. aoqi@0: // Only that thread can complete inflation -- other threads must wait. aoqi@0: // The INFLATING value is transient. aoqi@0: // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish. aoqi@0: // We could always eliminate polling by parking the thread on some auxiliary list. aoqi@0: if (mark == markOopDesc::INFLATING()) { aoqi@0: TEVENT (Inflate: spin while INFLATING) ; aoqi@0: ReadStableMark(object) ; aoqi@0: continue ; aoqi@0: } aoqi@0: aoqi@0: // CASE: stack-locked aoqi@0: // Could be stack-locked either by this thread or by some other thread. aoqi@0: // aoqi@0: // Note that we allocate the objectmonitor speculatively, _before_ attempting aoqi@0: // to install INFLATING into the mark word. We originally installed INFLATING, aoqi@0: // allocated the objectmonitor, and then finally STed the address of the aoqi@0: // objectmonitor into the mark. This was correct, but artificially lengthened aoqi@0: // the interval in which INFLATED appeared in the mark, thus increasing aoqi@0: // the odds of inflation contention. aoqi@0: // aoqi@0: // We now use per-thread private objectmonitor free lists. aoqi@0: // These list are reprovisioned from the global free list outside the aoqi@0: // critical INFLATING...ST interval. A thread can transfer aoqi@0: // multiple objectmonitors en-mass from the global free list to its local free list. aoqi@0: // This reduces coherency traffic and lock contention on the global free list. aoqi@0: // Using such local free lists, it doesn't matter if the omAlloc() call appears aoqi@0: // before or after the CAS(INFLATING) operation. aoqi@0: // See the comments in omAlloc(). aoqi@0: aoqi@0: if (mark->has_locker()) { aoqi@0: ObjectMonitor * m = omAlloc (Self) ; aoqi@0: // Optimistically prepare the objectmonitor - anticipate successful CAS aoqi@0: // We do this before the CAS in order to minimize the length of time aoqi@0: // in which INFLATING appears in the mark. aoqi@0: m->Recycle(); aoqi@0: m->_Responsible = NULL ; aoqi@0: m->OwnerIsThread = 0 ; aoqi@0: m->_recursions = 0 ; aoqi@0: m->_SpinDuration = ObjectMonitor::Knob_SpinLimit ; // Consider: maintain by type/class aoqi@0: aoqi@0: markOop cmp = (markOop) Atomic::cmpxchg_ptr (markOopDesc::INFLATING(), object->mark_addr(), mark) ; aoqi@0: if (cmp != mark) { aoqi@0: omRelease (Self, m, true) ; aoqi@0: continue ; // Interference -- just retry aoqi@0: } aoqi@0: aoqi@0: // We've successfully installed INFLATING (0) into the mark-word. aoqi@0: // This is the only case where 0 will appear in a mark-work. aoqi@0: // Only the singular thread that successfully swings the mark-word aoqi@0: // to 0 can perform (or more precisely, complete) inflation. aoqi@0: // aoqi@0: // Why do we CAS a 0 into the mark-word instead of just CASing the aoqi@0: // mark-word from the stack-locked value directly to the new inflated state? aoqi@0: // Consider what happens when a thread unlocks a stack-locked object. aoqi@0: // It attempts to use CAS to swing the displaced header value from the aoqi@0: // on-stack basiclock back into the object header. Recall also that the aoqi@0: // header value (hashcode, etc) can reside in (a) the object header, or aoqi@0: // (b) a displaced header associated with the stack-lock, or (c) a displaced aoqi@0: // header in an objectMonitor. The inflate() routine must copy the header aoqi@0: // value from the basiclock on the owner's stack to the objectMonitor, all aoqi@0: // the while preserving the hashCode stability invariants. If the owner aoqi@0: // decides to release the lock while the value is 0, the unlock will fail aoqi@0: // and control will eventually pass from slow_exit() to inflate. The owner aoqi@0: // will then spin, waiting for the 0 value to disappear. Put another way, aoqi@0: // the 0 causes the owner to stall if the owner happens to try to aoqi@0: // drop the lock (restoring the header from the basiclock to the object) aoqi@0: // while inflation is in-progress. This protocol avoids races that might aoqi@0: // would otherwise permit hashCode values to change or "flicker" for an object. aoqi@0: // Critically, while object->mark is 0 mark->displaced_mark_helper() is stable. aoqi@0: // 0 serves as a "BUSY" inflate-in-progress indicator. aoqi@0: aoqi@0: aoqi@0: // fetch the displaced mark from the owner's stack. aoqi@0: // The owner can't die or unwind past the lock while our INFLATING aoqi@0: // object is in the mark. Furthermore the owner can't complete aoqi@0: // an unlock on the object, either. aoqi@0: markOop dmw = mark->displaced_mark_helper() ; aoqi@0: assert (dmw->is_neutral(), "invariant") ; aoqi@0: aoqi@0: // Setup monitor fields to proper values -- prepare the monitor aoqi@0: m->set_header(dmw) ; aoqi@0: aoqi@0: // Optimization: if the mark->locker stack address is associated aoqi@0: // with this thread we could simply set m->_owner = Self and aoqi@0: // m->OwnerIsThread = 1. Note that a thread can inflate an object aoqi@0: // that it has stack-locked -- as might happen in wait() -- directly aoqi@0: // with CAS. That is, we can avoid the xchg-NULL .... ST idiom. aoqi@0: m->set_owner(mark->locker()); aoqi@0: m->set_object(object); aoqi@0: // TODO-FIXME: assert BasicLock->dhw != 0. aoqi@0: aoqi@0: // Must preserve store ordering. The monitor state must aoqi@0: // be stable at the time of publishing the monitor address. aoqi@0: guarantee (object->mark() == markOopDesc::INFLATING(), "invariant") ; aoqi@0: object->release_set_mark(markOopDesc::encode(m)); aoqi@0: aoqi@0: // Hopefully the performance counters are allocated on distinct cache lines aoqi@0: // to avoid false sharing on MP systems ... aoqi@0: if (ObjectMonitor::_sync_Inflations != NULL) ObjectMonitor::_sync_Inflations->inc() ; aoqi@0: TEVENT(Inflate: overwrite stacklock) ; aoqi@0: if (TraceMonitorInflation) { aoqi@0: if (object->is_instance()) { aoqi@0: ResourceMark rm; aoqi@0: tty->print_cr("Inflating object " INTPTR_FORMAT " , mark " INTPTR_FORMAT " , type %s", aoqi@0: (void *) object, (intptr_t) object->mark(), aoqi@0: object->klass()->external_name()); aoqi@0: } aoqi@0: } aoqi@0: return m ; aoqi@0: } aoqi@0: aoqi@0: // CASE: neutral aoqi@0: // TODO-FIXME: for entry we currently inflate and then try to CAS _owner. aoqi@0: // If we know we're inflating for entry it's better to inflate by swinging a aoqi@0: // pre-locked objectMonitor pointer into the object header. A successful aoqi@0: // CAS inflates the object *and* confers ownership to the inflating thread. aoqi@0: // In the current implementation we use a 2-step mechanism where we CAS() aoqi@0: // to inflate and then CAS() again to try to swing _owner from NULL to Self. aoqi@0: // An inflateTry() method that we could call from fast_enter() and slow_enter() aoqi@0: // would be useful. aoqi@0: aoqi@0: assert (mark->is_neutral(), "invariant"); aoqi@0: ObjectMonitor * m = omAlloc (Self) ; aoqi@0: // prepare m for installation - set monitor to initial state aoqi@0: m->Recycle(); aoqi@0: m->set_header(mark); aoqi@0: m->set_owner(NULL); aoqi@0: m->set_object(object); aoqi@0: m->OwnerIsThread = 1 ; aoqi@0: m->_recursions = 0 ; aoqi@0: m->_Responsible = NULL ; aoqi@0: m->_SpinDuration = ObjectMonitor::Knob_SpinLimit ; // consider: keep metastats by type/class aoqi@0: aoqi@0: if (Atomic::cmpxchg_ptr (markOopDesc::encode(m), object->mark_addr(), mark) != mark) { aoqi@0: m->set_object (NULL) ; aoqi@0: m->set_owner (NULL) ; aoqi@0: m->OwnerIsThread = 0 ; aoqi@0: m->Recycle() ; aoqi@0: omRelease (Self, m, true) ; aoqi@0: m = NULL ; aoqi@0: continue ; aoqi@0: // interference - the markword changed - just retry. aoqi@0: // The state-transitions are one-way, so there's no chance of aoqi@0: // live-lock -- "Inflated" is an absorbing state. aoqi@0: } aoqi@0: aoqi@0: // Hopefully the performance counters are allocated on distinct aoqi@0: // cache lines to avoid false sharing on MP systems ... aoqi@0: if (ObjectMonitor::_sync_Inflations != NULL) ObjectMonitor::_sync_Inflations->inc() ; aoqi@0: TEVENT(Inflate: overwrite neutral) ; aoqi@0: if (TraceMonitorInflation) { aoqi@0: if (object->is_instance()) { aoqi@0: ResourceMark rm; aoqi@0: tty->print_cr("Inflating object " INTPTR_FORMAT " , mark " INTPTR_FORMAT " , type %s", aoqi@0: (void *) object, (intptr_t) object->mark(), aoqi@0: object->klass()->external_name()); aoqi@0: } aoqi@0: } aoqi@0: return m ; aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: // Note that we could encounter some performance loss through false-sharing as aoqi@0: // multiple locks occupy the same $ line. Padding might be appropriate. aoqi@0: aoqi@0: aoqi@0: // Deflate_idle_monitors() is called at all safepoints, immediately aoqi@0: // after all mutators are stopped, but before any objects have moved. aoqi@0: // It traverses the list of known monitors, deflating where possible. aoqi@0: // The scavenged monitor are returned to the monitor free list. aoqi@0: // aoqi@0: // Beware that we scavenge at *every* stop-the-world point. aoqi@0: // Having a large number of monitors in-circulation negatively aoqi@0: // impacts the performance of some applications (e.g., PointBase). aoqi@0: // Broadly, we want to minimize the # of monitors in circulation. aoqi@0: // aoqi@0: // We have added a flag, MonitorInUseLists, which creates a list aoqi@0: // of active monitors for each thread. deflate_idle_monitors() aoqi@0: // only scans the per-thread inuse lists. omAlloc() puts all aoqi@0: // assigned monitors on the per-thread list. deflate_idle_monitors() aoqi@0: // returns the non-busy monitors to the global free list. aoqi@0: // When a thread dies, omFlush() adds the list of active monitors for aoqi@0: // that thread to a global gOmInUseList acquiring the aoqi@0: // global list lock. deflate_idle_monitors() acquires the global aoqi@0: // list lock to scan for non-busy monitors to the global free list. aoqi@0: // An alternative could have used a single global inuse list. The aoqi@0: // downside would have been the additional cost of acquiring the global list lock aoqi@0: // for every omAlloc(). aoqi@0: // aoqi@0: // Perversely, the heap size -- and thus the STW safepoint rate -- aoqi@0: // typically drives the scavenge rate. Large heaps can mean infrequent GC, aoqi@0: // which in turn can mean large(r) numbers of objectmonitors in circulation. aoqi@0: // This is an unfortunate aspect of this design. aoqi@0: // aoqi@0: aoqi@0: enum ManifestConstants { aoqi@0: ClearResponsibleAtSTW = 0, aoqi@0: MaximumRecheckInterval = 1000 aoqi@0: } ; aoqi@0: aoqi@0: // Deflate a single monitor if not in use aoqi@0: // Return true if deflated, false if in use aoqi@0: bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj, aoqi@0: ObjectMonitor** FreeHeadp, ObjectMonitor** FreeTailp) { aoqi@0: bool deflated; aoqi@0: // Normal case ... The monitor is associated with obj. aoqi@0: guarantee (obj->mark() == markOopDesc::encode(mid), "invariant") ; aoqi@0: guarantee (mid == obj->mark()->monitor(), "invariant"); aoqi@0: guarantee (mid->header()->is_neutral(), "invariant"); aoqi@0: aoqi@0: if (mid->is_busy()) { aoqi@0: if (ClearResponsibleAtSTW) mid->_Responsible = NULL ; aoqi@0: deflated = false; aoqi@0: } else { aoqi@0: // Deflate the monitor if it is no longer being used aoqi@0: // It's idle - scavenge and return to the global free list aoqi@0: // plain old deflation ... aoqi@0: TEVENT (deflate_idle_monitors - scavenge1) ; aoqi@0: if (TraceMonitorInflation) { aoqi@0: if (obj->is_instance()) { aoqi@0: ResourceMark rm; aoqi@0: tty->print_cr("Deflating object " INTPTR_FORMAT " , mark " INTPTR_FORMAT " , type %s", aoqi@0: (void *) obj, (intptr_t) obj->mark(), obj->klass()->external_name()); aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: // Restore the header back to obj aoqi@0: obj->release_set_mark(mid->header()); aoqi@0: mid->clear(); aoqi@0: aoqi@0: assert (mid->object() == NULL, "invariant") ; aoqi@0: aoqi@0: // Move the object to the working free list defined by FreeHead,FreeTail. aoqi@0: if (*FreeHeadp == NULL) *FreeHeadp = mid; aoqi@0: if (*FreeTailp != NULL) { aoqi@0: ObjectMonitor * prevtail = *FreeTailp; aoqi@0: assert(prevtail->FreeNext == NULL, "cleaned up deflated?"); // TODO KK aoqi@0: prevtail->FreeNext = mid; aoqi@0: } aoqi@0: *FreeTailp = mid; aoqi@0: deflated = true; aoqi@0: } aoqi@0: return deflated; aoqi@0: } aoqi@0: aoqi@0: // Caller acquires ListLock aoqi@0: int ObjectSynchronizer::walk_monitor_list(ObjectMonitor** listheadp, aoqi@0: ObjectMonitor** FreeHeadp, ObjectMonitor** FreeTailp) { aoqi@0: ObjectMonitor* mid; aoqi@0: ObjectMonitor* next; aoqi@0: ObjectMonitor* curmidinuse = NULL; aoqi@0: int deflatedcount = 0; aoqi@0: aoqi@0: for (mid = *listheadp; mid != NULL; ) { aoqi@0: oop obj = (oop) mid->object(); aoqi@0: bool deflated = false; aoqi@0: if (obj != NULL) { aoqi@0: deflated = deflate_monitor(mid, obj, FreeHeadp, FreeTailp); aoqi@0: } aoqi@0: if (deflated) { aoqi@0: // extract from per-thread in-use-list aoqi@0: if (mid == *listheadp) { aoqi@0: *listheadp = mid->FreeNext; aoqi@0: } else if (curmidinuse != NULL) { aoqi@0: curmidinuse->FreeNext = mid->FreeNext; // maintain the current thread inuselist aoqi@0: } aoqi@0: next = mid->FreeNext; aoqi@0: mid->FreeNext = NULL; // This mid is current tail in the FreeHead list aoqi@0: mid = next; aoqi@0: deflatedcount++; aoqi@0: } else { aoqi@0: curmidinuse = mid; aoqi@0: mid = mid->FreeNext; aoqi@0: } aoqi@0: } aoqi@0: return deflatedcount; aoqi@0: } aoqi@0: aoqi@0: void ObjectSynchronizer::deflate_idle_monitors() { aoqi@0: assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); aoqi@0: int nInuse = 0 ; // currently associated with objects aoqi@0: int nInCirculation = 0 ; // extant aoqi@0: int nScavenged = 0 ; // reclaimed aoqi@0: bool deflated = false; aoqi@0: aoqi@0: ObjectMonitor * FreeHead = NULL ; // Local SLL of scavenged monitors aoqi@0: ObjectMonitor * FreeTail = NULL ; aoqi@0: aoqi@0: TEVENT (deflate_idle_monitors) ; aoqi@0: // Prevent omFlush from changing mids in Thread dtor's during deflation aoqi@0: // And in case the vm thread is acquiring a lock during a safepoint aoqi@0: // See e.g. 6320749 aoqi@0: Thread::muxAcquire (&ListLock, "scavenge - return") ; aoqi@0: aoqi@0: if (MonitorInUseLists) { aoqi@0: int inUse = 0; aoqi@0: for (JavaThread* cur = Threads::first(); cur != NULL; cur = cur->next()) { aoqi@0: nInCirculation+= cur->omInUseCount; aoqi@0: int deflatedcount = walk_monitor_list(cur->omInUseList_addr(), &FreeHead, &FreeTail); aoqi@0: cur->omInUseCount-= deflatedcount; aoqi@0: // verifyInUse(cur); aoqi@0: nScavenged += deflatedcount; aoqi@0: nInuse += cur->omInUseCount; aoqi@0: } aoqi@0: aoqi@0: // For moribund threads, scan gOmInUseList aoqi@0: if (gOmInUseList) { aoqi@0: nInCirculation += gOmInUseCount; aoqi@0: int deflatedcount = walk_monitor_list((ObjectMonitor **)&gOmInUseList, &FreeHead, &FreeTail); aoqi@0: gOmInUseCount-= deflatedcount; aoqi@0: nScavenged += deflatedcount; aoqi@0: nInuse += gOmInUseCount; aoqi@0: } aoqi@0: aoqi@0: } else for (ObjectMonitor* block = gBlockList; block != NULL; block = next(block)) { aoqi@0: // Iterate over all extant monitors - Scavenge all idle monitors. aoqi@0: assert(block->object() == CHAINMARKER, "must be a block header"); aoqi@0: nInCirculation += _BLOCKSIZE ; aoqi@0: for (int i = 1 ; i < _BLOCKSIZE; i++) { aoqi@0: ObjectMonitor* mid = &block[i]; aoqi@0: oop obj = (oop) mid->object(); aoqi@0: aoqi@0: if (obj == NULL) { aoqi@0: // The monitor is not associated with an object. aoqi@0: // The monitor should either be a thread-specific private aoqi@0: // free list or the global free list. aoqi@0: // obj == NULL IMPLIES mid->is_busy() == 0 aoqi@0: guarantee (!mid->is_busy(), "invariant") ; aoqi@0: continue ; aoqi@0: } aoqi@0: deflated = deflate_monitor(mid, obj, &FreeHead, &FreeTail); aoqi@0: aoqi@0: if (deflated) { aoqi@0: mid->FreeNext = NULL ; aoqi@0: nScavenged ++ ; aoqi@0: } else { aoqi@0: nInuse ++; aoqi@0: } aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: MonitorFreeCount += nScavenged; aoqi@0: aoqi@0: // Consider: audit gFreeList to ensure that MonitorFreeCount and list agree. aoqi@0: aoqi@0: if (ObjectMonitor::Knob_Verbose) { aoqi@0: ::printf ("Deflate: InCirc=%d InUse=%d Scavenged=%d ForceMonitorScavenge=%d : pop=%d free=%d\n", aoqi@0: nInCirculation, nInuse, nScavenged, ForceMonitorScavenge, aoqi@0: MonitorPopulation, MonitorFreeCount) ; aoqi@0: ::fflush(stdout) ; aoqi@0: } aoqi@0: aoqi@0: ForceMonitorScavenge = 0; // Reset aoqi@0: aoqi@0: // Move the scavenged monitors back to the global free list. aoqi@0: if (FreeHead != NULL) { aoqi@0: guarantee (FreeTail != NULL && nScavenged > 0, "invariant") ; aoqi@0: assert (FreeTail->FreeNext == NULL, "invariant") ; aoqi@0: // constant-time list splice - prepend scavenged segment to gFreeList aoqi@0: FreeTail->FreeNext = gFreeList ; aoqi@0: gFreeList = FreeHead ; aoqi@0: } aoqi@0: Thread::muxRelease (&ListLock) ; aoqi@0: aoqi@0: if (ObjectMonitor::_sync_Deflations != NULL) ObjectMonitor::_sync_Deflations->inc(nScavenged) ; aoqi@0: if (ObjectMonitor::_sync_MonExtant != NULL) ObjectMonitor::_sync_MonExtant ->set_value(nInCirculation); aoqi@0: aoqi@0: // TODO: Add objectMonitor leak detection. aoqi@0: // Audit/inventory the objectMonitors -- make sure they're all accounted for. aoqi@0: GVars.stwRandom = os::random() ; aoqi@0: GVars.stwCycle ++ ; aoqi@0: } aoqi@0: aoqi@0: // Monitor cleanup on JavaThread::exit aoqi@0: aoqi@0: // Iterate through monitor cache and attempt to release thread's monitors aoqi@0: // Gives up on a particular monitor if an exception occurs, but continues aoqi@0: // the overall iteration, swallowing the exception. aoqi@0: class ReleaseJavaMonitorsClosure: public MonitorClosure { aoqi@0: private: aoqi@0: TRAPS; aoqi@0: aoqi@0: public: aoqi@0: ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {} aoqi@0: void do_monitor(ObjectMonitor* mid) { aoqi@0: if (mid->owner() == THREAD) { aoqi@0: (void)mid->complete_exit(CHECK); aoqi@0: } aoqi@0: } aoqi@0: }; aoqi@0: aoqi@0: // Release all inflated monitors owned by THREAD. Lightweight monitors are aoqi@0: // ignored. This is meant to be called during JNI thread detach which assumes aoqi@0: // all remaining monitors are heavyweight. All exceptions are swallowed. aoqi@0: // Scanning the extant monitor list can be time consuming. aoqi@0: // A simple optimization is to add a per-thread flag that indicates a thread aoqi@0: // called jni_monitorenter() during its lifetime. aoqi@0: // aoqi@0: // Instead of No_Savepoint_Verifier it might be cheaper to aoqi@0: // use an idiom of the form: aoqi@0: // auto int tmp = SafepointSynchronize::_safepoint_counter ; aoqi@0: // aoqi@0: // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ; aoqi@0: // Since the tests are extremely cheap we could leave them enabled aoqi@0: // for normal product builds. aoqi@0: aoqi@0: void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) { aoqi@0: assert(THREAD == JavaThread::current(), "must be current Java thread"); aoqi@0: No_Safepoint_Verifier nsv ; aoqi@0: ReleaseJavaMonitorsClosure rjmc(THREAD); aoqi@0: Thread::muxAcquire(&ListLock, "release_monitors_owned_by_thread"); aoqi@0: ObjectSynchronizer::monitors_iterate(&rjmc); aoqi@0: Thread::muxRelease(&ListLock); aoqi@0: THREAD->clear_pending_exception(); aoqi@0: } aoqi@0: aoqi@0: //------------------------------------------------------------------------------ aoqi@0: // Non-product code aoqi@0: aoqi@0: #ifndef PRODUCT aoqi@0: aoqi@0: // Verify all monitors in the monitor cache, the verification is weak. aoqi@0: void ObjectSynchronizer::verify() { aoqi@0: ObjectMonitor* block = gBlockList; aoqi@0: ObjectMonitor* mid; aoqi@0: while (block) { aoqi@0: assert(block->object() == CHAINMARKER, "must be a block header"); aoqi@0: for (int i = 1; i < _BLOCKSIZE; i++) { aoqi@0: mid = block + i; aoqi@0: oop object = (oop) mid->object(); aoqi@0: if (object != NULL) { aoqi@0: mid->verify(); aoqi@0: } aoqi@0: } aoqi@0: block = (ObjectMonitor*) block->FreeNext; aoqi@0: } aoqi@0: } aoqi@0: aoqi@0: // Check if monitor belongs to the monitor cache aoqi@0: // The list is grow-only so it's *relatively* safe to traverse aoqi@0: // the list of extant blocks without taking a lock. aoqi@0: aoqi@0: int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) { aoqi@0: ObjectMonitor* block = gBlockList; aoqi@0: aoqi@0: while (block) { aoqi@0: assert(block->object() == CHAINMARKER, "must be a block header"); aoqi@0: if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) { aoqi@0: address mon = (address) monitor; aoqi@0: address blk = (address) block; aoqi@0: size_t diff = mon - blk; aoqi@0: assert((diff % sizeof(ObjectMonitor)) == 0, "check"); aoqi@0: return 1; aoqi@0: } aoqi@0: block = (ObjectMonitor*) block->FreeNext; aoqi@0: } aoqi@0: return 0; aoqi@0: } aoqi@0: aoqi@0: #endif