1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Sat Dec 01 00:00:00 2007 +0000
1.3 @@ -0,0 +1,8787 @@
1.4 +/*
1.5 + * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
1.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
1.7 + *
1.8 + * This code is free software; you can redistribute it and/or modify it
1.9 + * under the terms of the GNU General Public License version 2 only, as
1.10 + * published by the Free Software Foundation.
1.11 + *
1.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
1.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1.14 + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
1.15 + * version 2 for more details (a copy is included in the LICENSE file that
1.16 + * accompanied this code).
1.17 + *
1.18 + * You should have received a copy of the GNU General Public License version
1.19 + * 2 along with this work; if not, write to the Free Software Foundation,
1.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
1.21 + *
1.22 + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
1.23 + * CA 95054 USA or visit www.sun.com if you need additional information or
1.24 + * have any questions.
1.25 + *
1.26 + */
1.27 +
1.28 +# include "incls/_precompiled.incl"
1.29 +# include "incls/_concurrentMarkSweepGeneration.cpp.incl"
1.30 +
1.31 +// statics
1.32 +CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;
1.33 +bool CMSCollector::_full_gc_requested = false;
1.34 +
1.35 +//////////////////////////////////////////////////////////////////
1.36 +// In support of CMS/VM thread synchronization
1.37 +//////////////////////////////////////////////////////////////////
1.38 +// We split use of the CGC_lock into 2 "levels".
1.39 +// The low-level locking is of the usual CGC_lock monitor. We introduce
1.40 +// a higher level "token" (hereafter "CMS token") built on top of the
1.41 +// low level monitor (hereafter "CGC lock").
1.42 +// The token-passing protocol gives priority to the VM thread. The
1.43 +// CMS-lock doesn't provide any fairness guarantees, but clients
1.44 +// should ensure that it is only held for very short, bounded
1.45 +// durations.
1.46 +//
1.47 +// When either of the CMS thread or the VM thread is involved in
1.48 +// collection operations during which it does not want the other
1.49 +// thread to interfere, it obtains the CMS token.
1.50 +//
1.51 +// If either thread tries to get the token while the other has
1.52 +// it, that thread waits. However, if the VM thread and CMS thread
1.53 +// both want the token, then the VM thread gets priority while the
1.54 +// CMS thread waits. This ensures, for instance, that the "concurrent"
1.55 +// phases of the CMS thread's work do not block out the VM thread
1.56 +// for long periods of time as the CMS thread continues to hog
1.57 +// the token. (See bug 4616232).
1.58 +//
1.59 +// The baton-passing functions are, however, controlled by the
1.60 +// flags _foregroundGCShouldWait and _foregroundGCIsActive,
1.61 +// and here the low-level CMS lock, not the high level token,
1.62 +// ensures mutual exclusion.
1.63 +//
1.64 +// Two important conditions that we have to satisfy:
1.65 +// 1. if a thread does a low-level wait on the CMS lock, then it
1.66 +// relinquishes the CMS token if it were holding that token
1.67 +// when it acquired the low-level CMS lock.
1.68 +// 2. any low-level notifications on the low-level lock
1.69 +// should only be sent when a thread has relinquished the token.
1.70 +//
1.71 +// In the absence of either property, we'd have potential deadlock.
1.72 +//
1.73 +// We protect each of the CMS (concurrent and sequential) phases
1.74 +// with the CMS _token_, not the CMS _lock_.
1.75 +//
1.76 +// The only code protected by CMS lock is the token acquisition code
1.77 +// itself, see ConcurrentMarkSweepThread::[de]synchronize(), and the
1.78 +// baton-passing code.
1.79 +//
1.80 +// Unfortunately, i couldn't come up with a good abstraction to factor and
1.81 +// hide the naked CGC_lock manipulation in the baton-passing code
1.82 +// further below. That's something we should try to do. Also, the proof
1.83 +// of correctness of this 2-level locking scheme is far from obvious,
1.84 +// and potentially quite slippery. We have an uneasy supsicion, for instance,
1.85 +// that there may be a theoretical possibility of delay/starvation in the
1.86 +// low-level lock/wait/notify scheme used for the baton-passing because of
1.87 +// potential intereference with the priority scheme embodied in the
1.88 +// CMS-token-passing protocol. See related comments at a CGC_lock->wait()
1.89 +// invocation further below and marked with "XXX 20011219YSR".
1.90 +// Indeed, as we note elsewhere, this may become yet more slippery
1.91 +// in the presence of multiple CMS and/or multiple VM threads. XXX
1.92 +
1.93 +class CMSTokenSync: public StackObj {
1.94 + private:
1.95 + bool _is_cms_thread;
1.96 + public:
1.97 + CMSTokenSync(bool is_cms_thread):
1.98 + _is_cms_thread(is_cms_thread) {
1.99 + assert(is_cms_thread == Thread::current()->is_ConcurrentGC_thread(),
1.100 + "Incorrect argument to constructor");
1.101 + ConcurrentMarkSweepThread::synchronize(_is_cms_thread);
1.102 + }
1.103 +
1.104 + ~CMSTokenSync() {
1.105 + assert(_is_cms_thread ?
1.106 + ConcurrentMarkSweepThread::cms_thread_has_cms_token() :
1.107 + ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1.108 + "Incorrect state");
1.109 + ConcurrentMarkSweepThread::desynchronize(_is_cms_thread);
1.110 + }
1.111 +};
1.112 +
1.113 +// Convenience class that does a CMSTokenSync, and then acquires
1.114 +// upto three locks.
1.115 +class CMSTokenSyncWithLocks: public CMSTokenSync {
1.116 + private:
1.117 + // Note: locks are acquired in textual declaration order
1.118 + // and released in the opposite order
1.119 + MutexLockerEx _locker1, _locker2, _locker3;
1.120 + public:
1.121 + CMSTokenSyncWithLocks(bool is_cms_thread, Mutex* mutex1,
1.122 + Mutex* mutex2 = NULL, Mutex* mutex3 = NULL):
1.123 + CMSTokenSync(is_cms_thread),
1.124 + _locker1(mutex1, Mutex::_no_safepoint_check_flag),
1.125 + _locker2(mutex2, Mutex::_no_safepoint_check_flag),
1.126 + _locker3(mutex3, Mutex::_no_safepoint_check_flag)
1.127 + { }
1.128 +};
1.129 +
1.130 +
1.131 +// Wrapper class to temporarily disable icms during a foreground cms collection.
1.132 +class ICMSDisabler: public StackObj {
1.133 + public:
1.134 + // The ctor disables icms and wakes up the thread so it notices the change;
1.135 + // the dtor re-enables icms. Note that the CMSCollector methods will check
1.136 + // CMSIncrementalMode.
1.137 + ICMSDisabler() { CMSCollector::disable_icms(); CMSCollector::start_icms(); }
1.138 + ~ICMSDisabler() { CMSCollector::enable_icms(); }
1.139 +};
1.140 +
1.141 +//////////////////////////////////////////////////////////////////
1.142 +// Concurrent Mark-Sweep Generation /////////////////////////////
1.143 +//////////////////////////////////////////////////////////////////
1.144 +
1.145 +NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;)
1.146 +
1.147 +// This struct contains per-thread things necessary to support parallel
1.148 +// young-gen collection.
1.149 +class CMSParGCThreadState: public CHeapObj {
1.150 + public:
1.151 + CFLS_LAB lab;
1.152 + PromotionInfo promo;
1.153 +
1.154 + // Constructor.
1.155 + CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) {
1.156 + promo.setSpace(cfls);
1.157 + }
1.158 +};
1.159 +
1.160 +ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration(
1.161 + ReservedSpace rs, size_t initial_byte_size, int level,
1.162 + CardTableRS* ct, bool use_adaptive_freelists,
1.163 + FreeBlockDictionary::DictionaryChoice dictionaryChoice) :
1.164 + CardGeneration(rs, initial_byte_size, level, ct),
1.165 + _dilatation_factor(((double)MinChunkSize)/((double)(oopDesc::header_size()))),
1.166 + _debug_collection_type(Concurrent_collection_type)
1.167 +{
1.168 + HeapWord* bottom = (HeapWord*) _virtual_space.low();
1.169 + HeapWord* end = (HeapWord*) _virtual_space.high();
1.170 +
1.171 + _direct_allocated_words = 0;
1.172 + NOT_PRODUCT(
1.173 + _numObjectsPromoted = 0;
1.174 + _numWordsPromoted = 0;
1.175 + _numObjectsAllocated = 0;
1.176 + _numWordsAllocated = 0;
1.177 + )
1.178 +
1.179 + _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end),
1.180 + use_adaptive_freelists,
1.181 + dictionaryChoice);
1.182 + NOT_PRODUCT(debug_cms_space = _cmsSpace;)
1.183 + if (_cmsSpace == NULL) {
1.184 + vm_exit_during_initialization(
1.185 + "CompactibleFreeListSpace allocation failure");
1.186 + }
1.187 + _cmsSpace->_gen = this;
1.188 +
1.189 + _gc_stats = new CMSGCStats();
1.190 +
1.191 + // Verify the assumption that FreeChunk::_prev and OopDesc::_klass
1.192 + // offsets match. The ability to tell free chunks from objects
1.193 + // depends on this property.
1.194 + debug_only(
1.195 + FreeChunk* junk = NULL;
1.196 + assert(junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
1.197 + "Offset of FreeChunk::_prev within FreeChunk must match"
1.198 + " that of OopDesc::_klass within OopDesc");
1.199 + )
1.200 + if (ParallelGCThreads > 0) {
1.201 + typedef CMSParGCThreadState* CMSParGCThreadStatePtr;
1.202 + _par_gc_thread_states =
1.203 + NEW_C_HEAP_ARRAY(CMSParGCThreadStatePtr, ParallelGCThreads);
1.204 + if (_par_gc_thread_states == NULL) {
1.205 + vm_exit_during_initialization("Could not allocate par gc structs");
1.206 + }
1.207 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.208 + _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace());
1.209 + if (_par_gc_thread_states[i] == NULL) {
1.210 + vm_exit_during_initialization("Could not allocate par gc structs");
1.211 + }
1.212 + }
1.213 + } else {
1.214 + _par_gc_thread_states = NULL;
1.215 + }
1.216 + _incremental_collection_failed = false;
1.217 + // The "dilatation_factor" is the expansion that can occur on
1.218 + // account of the fact that the minimum object size in the CMS
1.219 + // generation may be larger than that in, say, a contiguous young
1.220 + // generation.
1.221 + // Ideally, in the calculation below, we'd compute the dilatation
1.222 + // factor as: MinChunkSize/(promoting_gen's min object size)
1.223 + // Since we do not have such a general query interface for the
1.224 + // promoting generation, we'll instead just use the mimimum
1.225 + // object size (which today is a header's worth of space);
1.226 + // note that all arithmetic is in units of HeapWords.
1.227 + assert(MinChunkSize >= oopDesc::header_size(), "just checking");
1.228 + assert(_dilatation_factor >= 1.0, "from previous assert");
1.229 +}
1.230 +
1.231 +void ConcurrentMarkSweepGeneration::ref_processor_init() {
1.232 + assert(collector() != NULL, "no collector");
1.233 + collector()->ref_processor_init();
1.234 +}
1.235 +
1.236 +void CMSCollector::ref_processor_init() {
1.237 + if (_ref_processor == NULL) {
1.238 + // Allocate and initialize a reference processor
1.239 + _ref_processor = ReferenceProcessor::create_ref_processor(
1.240 + _span, // span
1.241 + _cmsGen->refs_discovery_is_atomic(), // atomic_discovery
1.242 + _cmsGen->refs_discovery_is_mt(), // mt_discovery
1.243 + &_is_alive_closure,
1.244 + ParallelGCThreads,
1.245 + ParallelRefProcEnabled);
1.246 + // Initialize the _ref_processor field of CMSGen
1.247 + _cmsGen->set_ref_processor(_ref_processor);
1.248 +
1.249 + // Allocate a dummy ref processor for perm gen.
1.250 + ReferenceProcessor* rp2 = new ReferenceProcessor();
1.251 + if (rp2 == NULL) {
1.252 + vm_exit_during_initialization("Could not allocate ReferenceProcessor object");
1.253 + }
1.254 + _permGen->set_ref_processor(rp2);
1.255 + }
1.256 +}
1.257 +
1.258 +CMSAdaptiveSizePolicy* CMSCollector::size_policy() {
1.259 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.260 + assert(gch->kind() == CollectedHeap::GenCollectedHeap,
1.261 + "Wrong type of heap");
1.262 + CMSAdaptiveSizePolicy* sp = (CMSAdaptiveSizePolicy*)
1.263 + gch->gen_policy()->size_policy();
1.264 + assert(sp->is_gc_cms_adaptive_size_policy(),
1.265 + "Wrong type of size policy");
1.266 + return sp;
1.267 +}
1.268 +
1.269 +CMSGCAdaptivePolicyCounters* CMSCollector::gc_adaptive_policy_counters() {
1.270 + CMSGCAdaptivePolicyCounters* results =
1.271 + (CMSGCAdaptivePolicyCounters*) collector_policy()->counters();
1.272 + assert(
1.273 + results->kind() == GCPolicyCounters::CMSGCAdaptivePolicyCountersKind,
1.274 + "Wrong gc policy counter kind");
1.275 + return results;
1.276 +}
1.277 +
1.278 +
1.279 +void ConcurrentMarkSweepGeneration::initialize_performance_counters() {
1.280 +
1.281 + const char* gen_name = "old";
1.282 +
1.283 + // Generation Counters - generation 1, 1 subspace
1.284 + _gen_counters = new GenerationCounters(gen_name, 1, 1, &_virtual_space);
1.285 +
1.286 + _space_counters = new GSpaceCounters(gen_name, 0,
1.287 + _virtual_space.reserved_size(),
1.288 + this, _gen_counters);
1.289 +}
1.290 +
1.291 +CMSStats::CMSStats(ConcurrentMarkSweepGeneration* cms_gen, unsigned int alpha):
1.292 + _cms_gen(cms_gen)
1.293 +{
1.294 + assert(alpha <= 100, "bad value");
1.295 + _saved_alpha = alpha;
1.296 +
1.297 + // Initialize the alphas to the bootstrap value of 100.
1.298 + _gc0_alpha = _cms_alpha = 100;
1.299 +
1.300 + _cms_begin_time.update();
1.301 + _cms_end_time.update();
1.302 +
1.303 + _gc0_duration = 0.0;
1.304 + _gc0_period = 0.0;
1.305 + _gc0_promoted = 0;
1.306 +
1.307 + _cms_duration = 0.0;
1.308 + _cms_period = 0.0;
1.309 + _cms_allocated = 0;
1.310 +
1.311 + _cms_used_at_gc0_begin = 0;
1.312 + _cms_used_at_gc0_end = 0;
1.313 + _allow_duty_cycle_reduction = false;
1.314 + _valid_bits = 0;
1.315 + _icms_duty_cycle = CMSIncrementalDutyCycle;
1.316 +}
1.317 +
1.318 +// If promotion failure handling is on use
1.319 +// the padded average size of the promotion for each
1.320 +// young generation collection.
1.321 +double CMSStats::time_until_cms_gen_full() const {
1.322 + size_t cms_free = _cms_gen->cmsSpace()->free();
1.323 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.324 + size_t expected_promotion = gch->get_gen(0)->capacity();
1.325 + if (HandlePromotionFailure) {
1.326 + expected_promotion = MIN2(
1.327 + (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average(),
1.328 + expected_promotion);
1.329 + }
1.330 + if (cms_free > expected_promotion) {
1.331 + // Start a cms collection if there isn't enough space to promote
1.332 + // for the next minor collection. Use the padded average as
1.333 + // a safety factor.
1.334 + cms_free -= expected_promotion;
1.335 +
1.336 + // Adjust by the safety factor.
1.337 + double cms_free_dbl = (double)cms_free;
1.338 + cms_free_dbl = cms_free_dbl * (100.0 - CMSIncrementalSafetyFactor) / 100.0;
1.339 +
1.340 + if (PrintGCDetails && Verbose) {
1.341 + gclog_or_tty->print_cr("CMSStats::time_until_cms_gen_full: cms_free "
1.342 + SIZE_FORMAT " expected_promotion " SIZE_FORMAT,
1.343 + cms_free, expected_promotion);
1.344 + gclog_or_tty->print_cr(" cms_free_dbl %f cms_consumption_rate %f",
1.345 + cms_free_dbl, cms_consumption_rate() + 1.0);
1.346 + }
1.347 + // Add 1 in case the consumption rate goes to zero.
1.348 + return cms_free_dbl / (cms_consumption_rate() + 1.0);
1.349 + }
1.350 + return 0.0;
1.351 +}
1.352 +
1.353 +// Compare the duration of the cms collection to the
1.354 +// time remaining before the cms generation is empty.
1.355 +// Note that the time from the start of the cms collection
1.356 +// to the start of the cms sweep (less than the total
1.357 +// duration of the cms collection) can be used. This
1.358 +// has been tried and some applications experienced
1.359 +// promotion failures early in execution. This was
1.360 +// possibly because the averages were not accurate
1.361 +// enough at the beginning.
1.362 +double CMSStats::time_until_cms_start() const {
1.363 + // We add "gc0_period" to the "work" calculation
1.364 + // below because this query is done (mostly) at the
1.365 + // end of a scavenge, so we need to conservatively
1.366 + // account for that much possible delay
1.367 + // in the query so as to avoid concurrent mode failures
1.368 + // due to starting the collection just a wee bit too
1.369 + // late.
1.370 + double work = cms_duration() + gc0_period();
1.371 + double deadline = time_until_cms_gen_full();
1.372 + if (work > deadline) {
1.373 + if (Verbose && PrintGCDetails) {
1.374 + gclog_or_tty->print(
1.375 + " CMSCollector: collect because of anticipated promotion "
1.376 + "before full %3.7f + %3.7f > %3.7f ", cms_duration(),
1.377 + gc0_period(), time_until_cms_gen_full());
1.378 + }
1.379 + return 0.0;
1.380 + }
1.381 + return work - deadline;
1.382 +}
1.383 +
1.384 +// Return a duty cycle based on old_duty_cycle and new_duty_cycle, limiting the
1.385 +// amount of change to prevent wild oscillation.
1.386 +unsigned int CMSStats::icms_damped_duty_cycle(unsigned int old_duty_cycle,
1.387 + unsigned int new_duty_cycle) {
1.388 + assert(old_duty_cycle <= 100, "bad input value");
1.389 + assert(new_duty_cycle <= 100, "bad input value");
1.390 +
1.391 + // Note: use subtraction with caution since it may underflow (values are
1.392 + // unsigned). Addition is safe since we're in the range 0-100.
1.393 + unsigned int damped_duty_cycle = new_duty_cycle;
1.394 + if (new_duty_cycle < old_duty_cycle) {
1.395 + const unsigned int largest_delta = MAX2(old_duty_cycle / 4, 5U);
1.396 + if (new_duty_cycle + largest_delta < old_duty_cycle) {
1.397 + damped_duty_cycle = old_duty_cycle - largest_delta;
1.398 + }
1.399 + } else if (new_duty_cycle > old_duty_cycle) {
1.400 + const unsigned int largest_delta = MAX2(old_duty_cycle / 4, 15U);
1.401 + if (new_duty_cycle > old_duty_cycle + largest_delta) {
1.402 + damped_duty_cycle = MIN2(old_duty_cycle + largest_delta, 100U);
1.403 + }
1.404 + }
1.405 + assert(damped_duty_cycle <= 100, "invalid duty cycle computed");
1.406 +
1.407 + if (CMSTraceIncrementalPacing) {
1.408 + gclog_or_tty->print(" [icms_damped_duty_cycle(%d,%d) = %d] ",
1.409 + old_duty_cycle, new_duty_cycle, damped_duty_cycle);
1.410 + }
1.411 + return damped_duty_cycle;
1.412 +}
1.413 +
1.414 +unsigned int CMSStats::icms_update_duty_cycle_impl() {
1.415 + assert(CMSIncrementalPacing && valid(),
1.416 + "should be handled in icms_update_duty_cycle()");
1.417 +
1.418 + double cms_time_so_far = cms_timer().seconds();
1.419 + double scaled_duration = cms_duration_per_mb() * _cms_used_at_gc0_end / M;
1.420 + double scaled_duration_remaining = fabsd(scaled_duration - cms_time_so_far);
1.421 +
1.422 + // Avoid division by 0.
1.423 + double time_until_full = MAX2(time_until_cms_gen_full(), 0.01);
1.424 + double duty_cycle_dbl = 100.0 * scaled_duration_remaining / time_until_full;
1.425 +
1.426 + unsigned int new_duty_cycle = MIN2((unsigned int)duty_cycle_dbl, 100U);
1.427 + if (new_duty_cycle > _icms_duty_cycle) {
1.428 + // Avoid very small duty cycles (1 or 2); 0 is allowed.
1.429 + if (new_duty_cycle > 2) {
1.430 + _icms_duty_cycle = icms_damped_duty_cycle(_icms_duty_cycle,
1.431 + new_duty_cycle);
1.432 + }
1.433 + } else if (_allow_duty_cycle_reduction) {
1.434 + // The duty cycle is reduced only once per cms cycle (see record_cms_end()).
1.435 + new_duty_cycle = icms_damped_duty_cycle(_icms_duty_cycle, new_duty_cycle);
1.436 + // Respect the minimum duty cycle.
1.437 + unsigned int min_duty_cycle = (unsigned int)CMSIncrementalDutyCycleMin;
1.438 + _icms_duty_cycle = MAX2(new_duty_cycle, min_duty_cycle);
1.439 + }
1.440 +
1.441 + if (PrintGCDetails || CMSTraceIncrementalPacing) {
1.442 + gclog_or_tty->print(" icms_dc=%d ", _icms_duty_cycle);
1.443 + }
1.444 +
1.445 + _allow_duty_cycle_reduction = false;
1.446 + return _icms_duty_cycle;
1.447 +}
1.448 +
1.449 +#ifndef PRODUCT
1.450 +void CMSStats::print_on(outputStream *st) const {
1.451 + st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);
1.452 + st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,
1.453 + gc0_duration(), gc0_period(), gc0_promoted());
1.454 + st->print(",cms_dur=%g,cms_dur_per_mb=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,
1.455 + cms_duration(), cms_duration_per_mb(),
1.456 + cms_period(), cms_allocated());
1.457 + st->print(",cms_since_beg=%g,cms_since_end=%g",
1.458 + cms_time_since_begin(), cms_time_since_end());
1.459 + st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,
1.460 + _cms_used_at_gc0_begin, _cms_used_at_gc0_end);
1.461 + if (CMSIncrementalMode) {
1.462 + st->print(",dc=%d", icms_duty_cycle());
1.463 + }
1.464 +
1.465 + if (valid()) {
1.466 + st->print(",promo_rate=%g,cms_alloc_rate=%g",
1.467 + promotion_rate(), cms_allocation_rate());
1.468 + st->print(",cms_consumption_rate=%g,time_until_full=%g",
1.469 + cms_consumption_rate(), time_until_cms_gen_full());
1.470 + }
1.471 + st->print(" ");
1.472 +}
1.473 +#endif // #ifndef PRODUCT
1.474 +
1.475 +CMSCollector::CollectorState CMSCollector::_collectorState =
1.476 + CMSCollector::Idling;
1.477 +bool CMSCollector::_foregroundGCIsActive = false;
1.478 +bool CMSCollector::_foregroundGCShouldWait = false;
1.479 +
1.480 +CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
1.481 + ConcurrentMarkSweepGeneration* permGen,
1.482 + CardTableRS* ct,
1.483 + ConcurrentMarkSweepPolicy* cp):
1.484 + _cmsGen(cmsGen),
1.485 + _permGen(permGen),
1.486 + _ct(ct),
1.487 + _ref_processor(NULL), // will be set later
1.488 + _conc_workers(NULL), // may be set later
1.489 + _abort_preclean(false),
1.490 + _start_sampling(false),
1.491 + _between_prologue_and_epilogue(false),
1.492 + _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
1.493 + _perm_gen_verify_bit_map(0, -1 /* no mutex */, "No_lock"),
1.494 + _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize),
1.495 + -1 /* lock-free */, "No_lock" /* dummy */),
1.496 + _modUnionClosure(&_modUnionTable),
1.497 + _modUnionClosurePar(&_modUnionTable),
1.498 + _is_alive_closure(&_markBitMap),
1.499 + _restart_addr(NULL),
1.500 + _overflow_list(NULL),
1.501 + _preserved_oop_stack(NULL),
1.502 + _preserved_mark_stack(NULL),
1.503 + _stats(cmsGen),
1.504 + _eden_chunk_array(NULL), // may be set in ctor body
1.505 + _eden_chunk_capacity(0), // -- ditto --
1.506 + _eden_chunk_index(0), // -- ditto --
1.507 + _survivor_plab_array(NULL), // -- ditto --
1.508 + _survivor_chunk_array(NULL), // -- ditto --
1.509 + _survivor_chunk_capacity(0), // -- ditto --
1.510 + _survivor_chunk_index(0), // -- ditto --
1.511 + _ser_pmc_preclean_ovflw(0),
1.512 + _ser_pmc_remark_ovflw(0),
1.513 + _par_pmc_remark_ovflw(0),
1.514 + _ser_kac_ovflw(0),
1.515 + _par_kac_ovflw(0),
1.516 +#ifndef PRODUCT
1.517 + _num_par_pushes(0),
1.518 +#endif
1.519 + _collection_count_start(0),
1.520 + _verifying(false),
1.521 + _icms_start_limit(NULL),
1.522 + _icms_stop_limit(NULL),
1.523 + _verification_mark_bm(0, Mutex::leaf + 1, "CMS_verification_mark_bm_lock"),
1.524 + _completed_initialization(false),
1.525 + _collector_policy(cp),
1.526 + _unload_classes(false),
1.527 + _unloaded_classes_last_cycle(false),
1.528 + _sweep_estimate(CMS_SweepWeight, CMS_SweepPadding)
1.529 +{
1.530 + if (ExplicitGCInvokesConcurrentAndUnloadsClasses) {
1.531 + ExplicitGCInvokesConcurrent = true;
1.532 + }
1.533 + // Now expand the span and allocate the collection support structures
1.534 + // (MUT, marking bit map etc.) to cover both generations subject to
1.535 + // collection.
1.536 +
1.537 + // First check that _permGen is adjacent to _cmsGen and above it.
1.538 + assert( _cmsGen->reserved().word_size() > 0
1.539 + && _permGen->reserved().word_size() > 0,
1.540 + "generations should not be of zero size");
1.541 + assert(_cmsGen->reserved().intersection(_permGen->reserved()).is_empty(),
1.542 + "_cmsGen and _permGen should not overlap");
1.543 + assert(_cmsGen->reserved().end() == _permGen->reserved().start(),
1.544 + "_cmsGen->end() different from _permGen->start()");
1.545 +
1.546 + // For use by dirty card to oop closures.
1.547 + _cmsGen->cmsSpace()->set_collector(this);
1.548 + _permGen->cmsSpace()->set_collector(this);
1.549 +
1.550 + // Adjust my span to cover old (cms) gen and perm gen
1.551 + _span = _cmsGen->reserved()._union(_permGen->reserved());
1.552 + // Initialize the span of is_alive_closure
1.553 + _is_alive_closure.set_span(_span);
1.554 +
1.555 + // Allocate MUT and marking bit map
1.556 + {
1.557 + MutexLockerEx x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag);
1.558 + if (!_markBitMap.allocate(_span)) {
1.559 + warning("Failed to allocate CMS Bit Map");
1.560 + return;
1.561 + }
1.562 + assert(_markBitMap.covers(_span), "_markBitMap inconsistency?");
1.563 + }
1.564 + {
1.565 + _modUnionTable.allocate(_span);
1.566 + assert(_modUnionTable.covers(_span), "_modUnionTable inconsistency?");
1.567 + }
1.568 +
1.569 + if (!_markStack.allocate(CMSMarkStackSize)) {
1.570 + warning("Failed to allocate CMS Marking Stack");
1.571 + return;
1.572 + }
1.573 + if (!_revisitStack.allocate(CMSRevisitStackSize)) {
1.574 + warning("Failed to allocate CMS Revisit Stack");
1.575 + return;
1.576 + }
1.577 +
1.578 + // Support for multi-threaded concurrent phases
1.579 + if (ParallelGCThreads > 0 && CMSConcurrentMTEnabled) {
1.580 + if (FLAG_IS_DEFAULT(ParallelCMSThreads)) {
1.581 + // just for now
1.582 + FLAG_SET_DEFAULT(ParallelCMSThreads, (ParallelGCThreads + 3)/4);
1.583 + }
1.584 + if (ParallelCMSThreads > 1) {
1.585 + _conc_workers = new YieldingFlexibleWorkGang("Parallel CMS Threads",
1.586 + ParallelCMSThreads, true);
1.587 + if (_conc_workers == NULL) {
1.588 + warning("GC/CMS: _conc_workers allocation failure: "
1.589 + "forcing -CMSConcurrentMTEnabled");
1.590 + CMSConcurrentMTEnabled = false;
1.591 + }
1.592 + } else {
1.593 + CMSConcurrentMTEnabled = false;
1.594 + }
1.595 + }
1.596 + if (!CMSConcurrentMTEnabled) {
1.597 + ParallelCMSThreads = 0;
1.598 + } else {
1.599 + // Turn off CMSCleanOnEnter optimization temporarily for
1.600 + // the MT case where it's not fixed yet; see 6178663.
1.601 + CMSCleanOnEnter = false;
1.602 + }
1.603 + assert((_conc_workers != NULL) == (ParallelCMSThreads > 1),
1.604 + "Inconsistency");
1.605 +
1.606 + // Parallel task queues; these are shared for the
1.607 + // concurrent and stop-world phases of CMS, but
1.608 + // are not shared with parallel scavenge (ParNew).
1.609 + {
1.610 + uint i;
1.611 + uint num_queues = (uint) MAX2(ParallelGCThreads, ParallelCMSThreads);
1.612 +
1.613 + if ((CMSParallelRemarkEnabled || CMSConcurrentMTEnabled
1.614 + || ParallelRefProcEnabled)
1.615 + && num_queues > 0) {
1.616 + _task_queues = new OopTaskQueueSet(num_queues);
1.617 + if (_task_queues == NULL) {
1.618 + warning("task_queues allocation failure.");
1.619 + return;
1.620 + }
1.621 + _hash_seed = NEW_C_HEAP_ARRAY(int, num_queues);
1.622 + if (_hash_seed == NULL) {
1.623 + warning("_hash_seed array allocation failure");
1.624 + return;
1.625 + }
1.626 +
1.627 + // XXX use a global constant instead of 64!
1.628 + typedef struct OopTaskQueuePadded {
1.629 + OopTaskQueue work_queue;
1.630 + char pad[64 - sizeof(OopTaskQueue)]; // prevent false sharing
1.631 + } OopTaskQueuePadded;
1.632 +
1.633 + for (i = 0; i < num_queues; i++) {
1.634 + OopTaskQueuePadded *q_padded = new OopTaskQueuePadded();
1.635 + if (q_padded == NULL) {
1.636 + warning("work_queue allocation failure.");
1.637 + return;
1.638 + }
1.639 + _task_queues->register_queue(i, &q_padded->work_queue);
1.640 + }
1.641 + for (i = 0; i < num_queues; i++) {
1.642 + _task_queues->queue(i)->initialize();
1.643 + _hash_seed[i] = 17; // copied from ParNew
1.644 + }
1.645 + }
1.646 + }
1.647 +
1.648 + // "initiatingOccupancy" is the occupancy ratio at which we trigger
1.649 + // a new collection cycle. Unless explicitly specified via
1.650 + // CMSTriggerRatio, it is calculated by:
1.651 + // Let "f" be MinHeapFreeRatio in
1.652 + //
1.653 + // intiatingOccupancy = 100-f +
1.654 + // f * (CMSTriggerRatio/100)
1.655 + // That is, if we assume the heap is at its desired maximum occupancy at the
1.656 + // end of a collection, we let CMSTriggerRatio of the (purported) free
1.657 + // space be allocated before initiating a new collection cycle.
1.658 + if (CMSInitiatingOccupancyFraction > 0) {
1.659 + _initiatingOccupancy = (double)CMSInitiatingOccupancyFraction / 100.0;
1.660 + } else {
1.661 + _initiatingOccupancy = ((100 - MinHeapFreeRatio) +
1.662 + (double)(CMSTriggerRatio *
1.663 + MinHeapFreeRatio) / 100.0)
1.664 + / 100.0;
1.665 + }
1.666 + // Clip CMSBootstrapOccupancy between 0 and 100.
1.667 + _bootstrap_occupancy = ((double)MIN2((intx)100, MAX2((intx)0, CMSBootstrapOccupancy)))
1.668 + /(double)100;
1.669 +
1.670 + _full_gcs_since_conc_gc = 0;
1.671 +
1.672 + // Now tell CMS generations the identity of their collector
1.673 + ConcurrentMarkSweepGeneration::set_collector(this);
1.674 +
1.675 + // Create & start a CMS thread for this CMS collector
1.676 + _cmsThread = ConcurrentMarkSweepThread::start(this);
1.677 + assert(cmsThread() != NULL, "CMS Thread should have been created");
1.678 + assert(cmsThread()->collector() == this,
1.679 + "CMS Thread should refer to this gen");
1.680 + assert(CGC_lock != NULL, "Where's the CGC_lock?");
1.681 +
1.682 + // Support for parallelizing young gen rescan
1.683 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.684 + _young_gen = gch->prev_gen(_cmsGen);
1.685 + if (gch->supports_inline_contig_alloc()) {
1.686 + _top_addr = gch->top_addr();
1.687 + _end_addr = gch->end_addr();
1.688 + assert(_young_gen != NULL, "no _young_gen");
1.689 + _eden_chunk_index = 0;
1.690 + _eden_chunk_capacity = (_young_gen->max_capacity()+CMSSamplingGrain)/CMSSamplingGrain;
1.691 + _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity);
1.692 + if (_eden_chunk_array == NULL) {
1.693 + _eden_chunk_capacity = 0;
1.694 + warning("GC/CMS: _eden_chunk_array allocation failure");
1.695 + }
1.696 + }
1.697 + assert(_eden_chunk_array != NULL || _eden_chunk_capacity == 0, "Error");
1.698 +
1.699 + // Support for parallelizing survivor space rescan
1.700 + if (CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) {
1.701 + size_t max_plab_samples = MaxNewSize/((SurvivorRatio+2)*MinTLABSize);
1.702 + _survivor_plab_array = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads);
1.703 + _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, 2*max_plab_samples);
1.704 + _cursor = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads);
1.705 + if (_survivor_plab_array == NULL || _survivor_chunk_array == NULL
1.706 + || _cursor == NULL) {
1.707 + warning("Failed to allocate survivor plab/chunk array");
1.708 + if (_survivor_plab_array != NULL) {
1.709 + FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array);
1.710 + _survivor_plab_array = NULL;
1.711 + }
1.712 + if (_survivor_chunk_array != NULL) {
1.713 + FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array);
1.714 + _survivor_chunk_array = NULL;
1.715 + }
1.716 + if (_cursor != NULL) {
1.717 + FREE_C_HEAP_ARRAY(size_t, _cursor);
1.718 + _cursor = NULL;
1.719 + }
1.720 + } else {
1.721 + _survivor_chunk_capacity = 2*max_plab_samples;
1.722 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.723 + HeapWord** vec = NEW_C_HEAP_ARRAY(HeapWord*, max_plab_samples);
1.724 + if (vec == NULL) {
1.725 + warning("Failed to allocate survivor plab array");
1.726 + for (int j = i; j > 0; j--) {
1.727 + FREE_C_HEAP_ARRAY(HeapWord*, _survivor_plab_array[j-1].array());
1.728 + }
1.729 + FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array);
1.730 + FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array);
1.731 + _survivor_plab_array = NULL;
1.732 + _survivor_chunk_array = NULL;
1.733 + _survivor_chunk_capacity = 0;
1.734 + break;
1.735 + } else {
1.736 + ChunkArray* cur =
1.737 + ::new (&_survivor_plab_array[i]) ChunkArray(vec,
1.738 + max_plab_samples);
1.739 + assert(cur->end() == 0, "Should be 0");
1.740 + assert(cur->array() == vec, "Should be vec");
1.741 + assert(cur->capacity() == max_plab_samples, "Error");
1.742 + }
1.743 + }
1.744 + }
1.745 + }
1.746 + assert( ( _survivor_plab_array != NULL
1.747 + && _survivor_chunk_array != NULL)
1.748 + || ( _survivor_chunk_capacity == 0
1.749 + && _survivor_chunk_index == 0),
1.750 + "Error");
1.751 +
1.752 + // Choose what strong roots should be scanned depending on verification options
1.753 + // and perm gen collection mode.
1.754 + if (!CMSClassUnloadingEnabled) {
1.755 + // If class unloading is disabled we want to include all classes into the root set.
1.756 + add_root_scanning_option(SharedHeap::SO_AllClasses);
1.757 + } else {
1.758 + add_root_scanning_option(SharedHeap::SO_SystemClasses);
1.759 + }
1.760 +
1.761 + NOT_PRODUCT(_overflow_counter = CMSMarkStackOverflowInterval;)
1.762 + _gc_counters = new CollectorCounters("CMS", 1);
1.763 + _completed_initialization = true;
1.764 + _sweep_timer.start(); // start of time
1.765 +}
1.766 +
1.767 +const char* ConcurrentMarkSweepGeneration::name() const {
1.768 + return "concurrent mark-sweep generation";
1.769 +}
1.770 +void ConcurrentMarkSweepGeneration::update_counters() {
1.771 + if (UsePerfData) {
1.772 + _space_counters->update_all();
1.773 + _gen_counters->update_all();
1.774 + }
1.775 +}
1.776 +
1.777 +// this is an optimized version of update_counters(). it takes the
1.778 +// used value as a parameter rather than computing it.
1.779 +//
1.780 +void ConcurrentMarkSweepGeneration::update_counters(size_t used) {
1.781 + if (UsePerfData) {
1.782 + _space_counters->update_used(used);
1.783 + _space_counters->update_capacity();
1.784 + _gen_counters->update_all();
1.785 + }
1.786 +}
1.787 +
1.788 +void ConcurrentMarkSweepGeneration::print() const {
1.789 + Generation::print();
1.790 + cmsSpace()->print();
1.791 +}
1.792 +
1.793 +#ifndef PRODUCT
1.794 +void ConcurrentMarkSweepGeneration::print_statistics() {
1.795 + cmsSpace()->printFLCensus(0);
1.796 +}
1.797 +#endif
1.798 +
1.799 +void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) {
1.800 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.801 + if (PrintGCDetails) {
1.802 + if (Verbose) {
1.803 + gclog_or_tty->print(" [%d %s-%s: "SIZE_FORMAT"("SIZE_FORMAT")]",
1.804 + level(), short_name(), s, used(), capacity());
1.805 + } else {
1.806 + gclog_or_tty->print(" [%d %s-%s: "SIZE_FORMAT"K("SIZE_FORMAT"K)]",
1.807 + level(), short_name(), s, used() / K, capacity() / K);
1.808 + }
1.809 + }
1.810 + if (Verbose) {
1.811 + gclog_or_tty->print(" "SIZE_FORMAT"("SIZE_FORMAT")",
1.812 + gch->used(), gch->capacity());
1.813 + } else {
1.814 + gclog_or_tty->print(" "SIZE_FORMAT"K("SIZE_FORMAT"K)",
1.815 + gch->used() / K, gch->capacity() / K);
1.816 + }
1.817 +}
1.818 +
1.819 +size_t
1.820 +ConcurrentMarkSweepGeneration::contiguous_available() const {
1.821 + // dld proposes an improvement in precision here. If the committed
1.822 + // part of the space ends in a free block we should add that to
1.823 + // uncommitted size in the calculation below. Will make this
1.824 + // change later, staying with the approximation below for the
1.825 + // time being. -- ysr.
1.826 + return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
1.827 +}
1.828 +
1.829 +size_t
1.830 +ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
1.831 + return _cmsSpace->max_alloc_in_words() * HeapWordSize;
1.832 +}
1.833 +
1.834 +size_t ConcurrentMarkSweepGeneration::max_available() const {
1.835 + return free() + _virtual_space.uncommitted_size();
1.836 +}
1.837 +
1.838 +bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(
1.839 + size_t max_promotion_in_bytes,
1.840 + bool younger_handles_promotion_failure) const {
1.841 +
1.842 + // This is the most conservative test. Full promotion is
1.843 + // guaranteed if this is used. The multiplicative factor is to
1.844 + // account for the worst case "dilatation".
1.845 + double adjusted_max_promo_bytes = _dilatation_factor * max_promotion_in_bytes;
1.846 + if (adjusted_max_promo_bytes > (double)max_uintx) { // larger than size_t
1.847 + adjusted_max_promo_bytes = (double)max_uintx;
1.848 + }
1.849 + bool result = (max_contiguous_available() >= (size_t)adjusted_max_promo_bytes);
1.850 +
1.851 + if (younger_handles_promotion_failure && !result) {
1.852 + // Full promotion is not guaranteed because fragmentation
1.853 + // of the cms generation can prevent the full promotion.
1.854 + result = (max_available() >= (size_t)adjusted_max_promo_bytes);
1.855 +
1.856 + if (!result) {
1.857 + // With promotion failure handling the test for the ability
1.858 + // to support the promotion does not have to be guaranteed.
1.859 + // Use an average of the amount promoted.
1.860 + result = max_available() >= (size_t)
1.861 + gc_stats()->avg_promoted()->padded_average();
1.862 + if (PrintGC && Verbose && result) {
1.863 + gclog_or_tty->print_cr(
1.864 + "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
1.865 + " max_available: " SIZE_FORMAT
1.866 + " avg_promoted: " SIZE_FORMAT,
1.867 + max_available(), (size_t)
1.868 + gc_stats()->avg_promoted()->padded_average());
1.869 + }
1.870 + } else {
1.871 + if (PrintGC && Verbose) {
1.872 + gclog_or_tty->print_cr(
1.873 + "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
1.874 + " max_available: " SIZE_FORMAT
1.875 + " adj_max_promo_bytes: " SIZE_FORMAT,
1.876 + max_available(), (size_t)adjusted_max_promo_bytes);
1.877 + }
1.878 + }
1.879 + } else {
1.880 + if (PrintGC && Verbose) {
1.881 + gclog_or_tty->print_cr(
1.882 + "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
1.883 + " contiguous_available: " SIZE_FORMAT
1.884 + " adj_max_promo_bytes: " SIZE_FORMAT,
1.885 + max_contiguous_available(), (size_t)adjusted_max_promo_bytes);
1.886 + }
1.887 + }
1.888 + return result;
1.889 +}
1.890 +
1.891 +CompactibleSpace*
1.892 +ConcurrentMarkSweepGeneration::first_compaction_space() const {
1.893 + return _cmsSpace;
1.894 +}
1.895 +
1.896 +void ConcurrentMarkSweepGeneration::reset_after_compaction() {
1.897 + // Clear the promotion information. These pointers can be adjusted
1.898 + // along with all the other pointers into the heap but
1.899 + // compaction is expected to be a rare event with
1.900 + // a heap using cms so don't do it without seeing the need.
1.901 + if (ParallelGCThreads > 0) {
1.902 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.903 + _par_gc_thread_states[i]->promo.reset();
1.904 + }
1.905 + }
1.906 +}
1.907 +
1.908 +void ConcurrentMarkSweepGeneration::space_iterate(SpaceClosure* blk, bool usedOnly) {
1.909 + blk->do_space(_cmsSpace);
1.910 +}
1.911 +
1.912 +void ConcurrentMarkSweepGeneration::compute_new_size() {
1.913 + assert_locked_or_safepoint(Heap_lock);
1.914 +
1.915 + // If incremental collection failed, we just want to expand
1.916 + // to the limit.
1.917 + if (incremental_collection_failed()) {
1.918 + clear_incremental_collection_failed();
1.919 + grow_to_reserved();
1.920 + return;
1.921 + }
1.922 +
1.923 + size_t expand_bytes = 0;
1.924 + double free_percentage = ((double) free()) / capacity();
1.925 + double desired_free_percentage = (double) MinHeapFreeRatio / 100;
1.926 + double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
1.927 +
1.928 + // compute expansion delta needed for reaching desired free percentage
1.929 + if (free_percentage < desired_free_percentage) {
1.930 + size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
1.931 + assert(desired_capacity >= capacity(), "invalid expansion size");
1.932 + expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
1.933 + }
1.934 + if (expand_bytes > 0) {
1.935 + if (PrintGCDetails && Verbose) {
1.936 + size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
1.937 + gclog_or_tty->print_cr("\nFrom compute_new_size: ");
1.938 + gclog_or_tty->print_cr(" Free fraction %f", free_percentage);
1.939 + gclog_or_tty->print_cr(" Desired free fraction %f",
1.940 + desired_free_percentage);
1.941 + gclog_or_tty->print_cr(" Maximum free fraction %f",
1.942 + maximum_free_percentage);
1.943 + gclog_or_tty->print_cr(" Capactiy "SIZE_FORMAT, capacity()/1000);
1.944 + gclog_or_tty->print_cr(" Desired capacity "SIZE_FORMAT,
1.945 + desired_capacity/1000);
1.946 + int prev_level = level() - 1;
1.947 + if (prev_level >= 0) {
1.948 + size_t prev_size = 0;
1.949 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.950 + Generation* prev_gen = gch->_gens[prev_level];
1.951 + prev_size = prev_gen->capacity();
1.952 + gclog_or_tty->print_cr(" Younger gen size "SIZE_FORMAT,
1.953 + prev_size/1000);
1.954 + }
1.955 + gclog_or_tty->print_cr(" unsafe_max_alloc_nogc "SIZE_FORMAT,
1.956 + unsafe_max_alloc_nogc()/1000);
1.957 + gclog_or_tty->print_cr(" contiguous available "SIZE_FORMAT,
1.958 + contiguous_available()/1000);
1.959 + gclog_or_tty->print_cr(" Expand by "SIZE_FORMAT" (bytes)",
1.960 + expand_bytes);
1.961 + }
1.962 + // safe if expansion fails
1.963 + expand(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
1.964 + if (PrintGCDetails && Verbose) {
1.965 + gclog_or_tty->print_cr(" Expanded free fraction %f",
1.966 + ((double) free()) / capacity());
1.967 + }
1.968 + }
1.969 +}
1.970 +
1.971 +Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
1.972 + return cmsSpace()->freelistLock();
1.973 +}
1.974 +
1.975 +HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size,
1.976 + bool tlab) {
1.977 + CMSSynchronousYieldRequest yr;
1.978 + MutexLockerEx x(freelistLock(),
1.979 + Mutex::_no_safepoint_check_flag);
1.980 + return have_lock_and_allocate(size, tlab);
1.981 +}
1.982 +
1.983 +HeapWord* ConcurrentMarkSweepGeneration::have_lock_and_allocate(size_t size,
1.984 + bool tlab) {
1.985 + assert_lock_strong(freelistLock());
1.986 + size_t adjustedSize = CompactibleFreeListSpace::adjustObjectSize(size);
1.987 + HeapWord* res = cmsSpace()->allocate(adjustedSize);
1.988 + // Allocate the object live (grey) if the background collector has
1.989 + // started marking. This is necessary because the marker may
1.990 + // have passed this address and consequently this object will
1.991 + // not otherwise be greyed and would be incorrectly swept up.
1.992 + // Note that if this object contains references, the writing
1.993 + // of those references will dirty the card containing this object
1.994 + // allowing the object to be blackened (and its references scanned)
1.995 + // either during a preclean phase or at the final checkpoint.
1.996 + if (res != NULL) {
1.997 + collector()->direct_allocated(res, adjustedSize);
1.998 + _direct_allocated_words += adjustedSize;
1.999 + // allocation counters
1.1000 + NOT_PRODUCT(
1.1001 + _numObjectsAllocated++;
1.1002 + _numWordsAllocated += (int)adjustedSize;
1.1003 + )
1.1004 + }
1.1005 + return res;
1.1006 +}
1.1007 +
1.1008 +// In the case of direct allocation by mutators in a generation that
1.1009 +// is being concurrently collected, the object must be allocated
1.1010 +// live (grey) if the background collector has started marking.
1.1011 +// This is necessary because the marker may
1.1012 +// have passed this address and consequently this object will
1.1013 +// not otherwise be greyed and would be incorrectly swept up.
1.1014 +// Note that if this object contains references, the writing
1.1015 +// of those references will dirty the card containing this object
1.1016 +// allowing the object to be blackened (and its references scanned)
1.1017 +// either during a preclean phase or at the final checkpoint.
1.1018 +void CMSCollector::direct_allocated(HeapWord* start, size_t size) {
1.1019 + assert(_markBitMap.covers(start, size), "Out of bounds");
1.1020 + if (_collectorState >= Marking) {
1.1021 + MutexLockerEx y(_markBitMap.lock(),
1.1022 + Mutex::_no_safepoint_check_flag);
1.1023 + // [see comments preceding SweepClosure::do_blk() below for details]
1.1024 + // 1. need to mark the object as live so it isn't collected
1.1025 + // 2. need to mark the 2nd bit to indicate the object may be uninitialized
1.1026 + // 3. need to mark the end of the object so sweeper can skip over it
1.1027 + // if it's uninitialized when the sweeper reaches it.
1.1028 + _markBitMap.mark(start); // object is live
1.1029 + _markBitMap.mark(start + 1); // object is potentially uninitialized?
1.1030 + _markBitMap.mark(start + size - 1);
1.1031 + // mark end of object
1.1032 + }
1.1033 + // check that oop looks uninitialized
1.1034 + assert(oop(start)->klass() == NULL, "_klass should be NULL");
1.1035 +}
1.1036 +
1.1037 +void CMSCollector::promoted(bool par, HeapWord* start,
1.1038 + bool is_obj_array, size_t obj_size) {
1.1039 + assert(_markBitMap.covers(start), "Out of bounds");
1.1040 + // See comment in direct_allocated() about when objects should
1.1041 + // be allocated live.
1.1042 + if (_collectorState >= Marking) {
1.1043 + // we already hold the marking bit map lock, taken in
1.1044 + // the prologue
1.1045 + if (par) {
1.1046 + _markBitMap.par_mark(start);
1.1047 + } else {
1.1048 + _markBitMap.mark(start);
1.1049 + }
1.1050 + // We don't need to mark the object as uninitialized (as
1.1051 + // in direct_allocated above) because this is being done with the
1.1052 + // world stopped and the object will be initialized by the
1.1053 + // time the sweeper gets to look at it.
1.1054 + assert(SafepointSynchronize::is_at_safepoint(),
1.1055 + "expect promotion only at safepoints");
1.1056 +
1.1057 + if (_collectorState < Sweeping) {
1.1058 + // Mark the appropriate cards in the modUnionTable, so that
1.1059 + // this object gets scanned before the sweep. If this is
1.1060 + // not done, CMS generation references in the object might
1.1061 + // not get marked.
1.1062 + // For the case of arrays, which are otherwise precisely
1.1063 + // marked, we need to dirty the entire array, not just its head.
1.1064 + if (is_obj_array) {
1.1065 + // The [par_]mark_range() method expects mr.end() below to
1.1066 + // be aligned to the granularity of a bit's representation
1.1067 + // in the heap. In the case of the MUT below, that's a
1.1068 + // card size.
1.1069 + MemRegion mr(start,
1.1070 + (HeapWord*)round_to((intptr_t)(start + obj_size),
1.1071 + CardTableModRefBS::card_size /* bytes */));
1.1072 + if (par) {
1.1073 + _modUnionTable.par_mark_range(mr);
1.1074 + } else {
1.1075 + _modUnionTable.mark_range(mr);
1.1076 + }
1.1077 + } else { // not an obj array; we can just mark the head
1.1078 + if (par) {
1.1079 + _modUnionTable.par_mark(start);
1.1080 + } else {
1.1081 + _modUnionTable.mark(start);
1.1082 + }
1.1083 + }
1.1084 + }
1.1085 + }
1.1086 +}
1.1087 +
1.1088 +static inline size_t percent_of_space(Space* space, HeapWord* addr)
1.1089 +{
1.1090 + size_t delta = pointer_delta(addr, space->bottom());
1.1091 + return (size_t)(delta * 100.0 / (space->capacity() / HeapWordSize));
1.1092 +}
1.1093 +
1.1094 +void CMSCollector::icms_update_allocation_limits()
1.1095 +{
1.1096 + Generation* gen0 = GenCollectedHeap::heap()->get_gen(0);
1.1097 + EdenSpace* eden = gen0->as_DefNewGeneration()->eden();
1.1098 +
1.1099 + const unsigned int duty_cycle = stats().icms_update_duty_cycle();
1.1100 + if (CMSTraceIncrementalPacing) {
1.1101 + stats().print();
1.1102 + }
1.1103 +
1.1104 + assert(duty_cycle <= 100, "invalid duty cycle");
1.1105 + if (duty_cycle != 0) {
1.1106 + // The duty_cycle is a percentage between 0 and 100; convert to words and
1.1107 + // then compute the offset from the endpoints of the space.
1.1108 + size_t free_words = eden->free() / HeapWordSize;
1.1109 + double free_words_dbl = (double)free_words;
1.1110 + size_t duty_cycle_words = (size_t)(free_words_dbl * duty_cycle / 100.0);
1.1111 + size_t offset_words = (free_words - duty_cycle_words) / 2;
1.1112 +
1.1113 + _icms_start_limit = eden->top() + offset_words;
1.1114 + _icms_stop_limit = eden->end() - offset_words;
1.1115 +
1.1116 + // The limits may be adjusted (shifted to the right) by
1.1117 + // CMSIncrementalOffset, to allow the application more mutator time after a
1.1118 + // young gen gc (when all mutators were stopped) and before CMS starts and
1.1119 + // takes away one or more cpus.
1.1120 + if (CMSIncrementalOffset != 0) {
1.1121 + double adjustment_dbl = free_words_dbl * CMSIncrementalOffset / 100.0;
1.1122 + size_t adjustment = (size_t)adjustment_dbl;
1.1123 + HeapWord* tmp_stop = _icms_stop_limit + adjustment;
1.1124 + if (tmp_stop > _icms_stop_limit && tmp_stop < eden->end()) {
1.1125 + _icms_start_limit += adjustment;
1.1126 + _icms_stop_limit = tmp_stop;
1.1127 + }
1.1128 + }
1.1129 + }
1.1130 + if (duty_cycle == 0 || (_icms_start_limit == _icms_stop_limit)) {
1.1131 + _icms_start_limit = _icms_stop_limit = eden->end();
1.1132 + }
1.1133 +
1.1134 + // Install the new start limit.
1.1135 + eden->set_soft_end(_icms_start_limit);
1.1136 +
1.1137 + if (CMSTraceIncrementalMode) {
1.1138 + gclog_or_tty->print(" icms alloc limits: "
1.1139 + PTR_FORMAT "," PTR_FORMAT
1.1140 + " (" SIZE_FORMAT "%%," SIZE_FORMAT "%%) ",
1.1141 + _icms_start_limit, _icms_stop_limit,
1.1142 + percent_of_space(eden, _icms_start_limit),
1.1143 + percent_of_space(eden, _icms_stop_limit));
1.1144 + if (Verbose) {
1.1145 + gclog_or_tty->print("eden: ");
1.1146 + eden->print_on(gclog_or_tty);
1.1147 + }
1.1148 + }
1.1149 +}
1.1150 +
1.1151 +// Any changes here should try to maintain the invariant
1.1152 +// that if this method is called with _icms_start_limit
1.1153 +// and _icms_stop_limit both NULL, then it should return NULL
1.1154 +// and not notify the icms thread.
1.1155 +HeapWord*
1.1156 +CMSCollector::allocation_limit_reached(Space* space, HeapWord* top,
1.1157 + size_t word_size)
1.1158 +{
1.1159 + // A start_limit equal to end() means the duty cycle is 0, so treat that as a
1.1160 + // nop.
1.1161 + if (CMSIncrementalMode && _icms_start_limit != space->end()) {
1.1162 + if (top <= _icms_start_limit) {
1.1163 + if (CMSTraceIncrementalMode) {
1.1164 + space->print_on(gclog_or_tty);
1.1165 + gclog_or_tty->stamp();
1.1166 + gclog_or_tty->print_cr(" start limit top=" PTR_FORMAT
1.1167 + ", new limit=" PTR_FORMAT
1.1168 + " (" SIZE_FORMAT "%%)",
1.1169 + top, _icms_stop_limit,
1.1170 + percent_of_space(space, _icms_stop_limit));
1.1171 + }
1.1172 + ConcurrentMarkSweepThread::start_icms();
1.1173 + assert(top < _icms_stop_limit, "Tautology");
1.1174 + if (word_size < pointer_delta(_icms_stop_limit, top)) {
1.1175 + return _icms_stop_limit;
1.1176 + }
1.1177 +
1.1178 + // The allocation will cross both the _start and _stop limits, so do the
1.1179 + // stop notification also and return end().
1.1180 + if (CMSTraceIncrementalMode) {
1.1181 + space->print_on(gclog_or_tty);
1.1182 + gclog_or_tty->stamp();
1.1183 + gclog_or_tty->print_cr(" +stop limit top=" PTR_FORMAT
1.1184 + ", new limit=" PTR_FORMAT
1.1185 + " (" SIZE_FORMAT "%%)",
1.1186 + top, space->end(),
1.1187 + percent_of_space(space, space->end()));
1.1188 + }
1.1189 + ConcurrentMarkSweepThread::stop_icms();
1.1190 + return space->end();
1.1191 + }
1.1192 +
1.1193 + if (top <= _icms_stop_limit) {
1.1194 + if (CMSTraceIncrementalMode) {
1.1195 + space->print_on(gclog_or_tty);
1.1196 + gclog_or_tty->stamp();
1.1197 + gclog_or_tty->print_cr(" stop limit top=" PTR_FORMAT
1.1198 + ", new limit=" PTR_FORMAT
1.1199 + " (" SIZE_FORMAT "%%)",
1.1200 + top, space->end(),
1.1201 + percent_of_space(space, space->end()));
1.1202 + }
1.1203 + ConcurrentMarkSweepThread::stop_icms();
1.1204 + return space->end();
1.1205 + }
1.1206 +
1.1207 + if (CMSTraceIncrementalMode) {
1.1208 + space->print_on(gclog_or_tty);
1.1209 + gclog_or_tty->stamp();
1.1210 + gclog_or_tty->print_cr(" end limit top=" PTR_FORMAT
1.1211 + ", new limit=" PTR_FORMAT,
1.1212 + top, NULL);
1.1213 + }
1.1214 + }
1.1215 +
1.1216 + return NULL;
1.1217 +}
1.1218 +
1.1219 +oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size, oop* ref) {
1.1220 + assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");
1.1221 + // allocate, copy and if necessary update promoinfo --
1.1222 + // delegate to underlying space.
1.1223 + assert_lock_strong(freelistLock());
1.1224 +
1.1225 +#ifndef PRODUCT
1.1226 + if (Universe::heap()->promotion_should_fail()) {
1.1227 + return NULL;
1.1228 + }
1.1229 +#endif // #ifndef PRODUCT
1.1230 +
1.1231 + oop res = _cmsSpace->promote(obj, obj_size, ref);
1.1232 + if (res == NULL) {
1.1233 + // expand and retry
1.1234 + size_t s = _cmsSpace->expansionSpaceRequired(obj_size); // HeapWords
1.1235 + expand(s*HeapWordSize, MinHeapDeltaBytes,
1.1236 + CMSExpansionCause::_satisfy_promotion);
1.1237 + // Since there's currently no next generation, we don't try to promote
1.1238 + // into a more senior generation.
1.1239 + assert(next_gen() == NULL, "assumption, based upon which no attempt "
1.1240 + "is made to pass on a possibly failing "
1.1241 + "promotion to next generation");
1.1242 + res = _cmsSpace->promote(obj, obj_size, ref);
1.1243 + }
1.1244 + if (res != NULL) {
1.1245 + // See comment in allocate() about when objects should
1.1246 + // be allocated live.
1.1247 + assert(obj->is_oop(), "Will dereference klass pointer below");
1.1248 + collector()->promoted(false, // Not parallel
1.1249 + (HeapWord*)res, obj->is_objArray(), obj_size);
1.1250 + // promotion counters
1.1251 + NOT_PRODUCT(
1.1252 + _numObjectsPromoted++;
1.1253 + _numWordsPromoted +=
1.1254 + (int)(CompactibleFreeListSpace::adjustObjectSize(obj->size()));
1.1255 + )
1.1256 + }
1.1257 + return res;
1.1258 +}
1.1259 +
1.1260 +
1.1261 +HeapWord*
1.1262 +ConcurrentMarkSweepGeneration::allocation_limit_reached(Space* space,
1.1263 + HeapWord* top,
1.1264 + size_t word_sz)
1.1265 +{
1.1266 + return collector()->allocation_limit_reached(space, top, word_sz);
1.1267 +}
1.1268 +
1.1269 +// Things to support parallel young-gen collection.
1.1270 +oop
1.1271 +ConcurrentMarkSweepGeneration::par_promote(int thread_num,
1.1272 + oop old, markOop m,
1.1273 + size_t word_sz) {
1.1274 +#ifndef PRODUCT
1.1275 + if (Universe::heap()->promotion_should_fail()) {
1.1276 + return NULL;
1.1277 + }
1.1278 +#endif // #ifndef PRODUCT
1.1279 +
1.1280 + CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1.1281 + PromotionInfo* promoInfo = &ps->promo;
1.1282 + // if we are tracking promotions, then first ensure space for
1.1283 + // promotion (including spooling space for saving header if necessary).
1.1284 + // then allocate and copy, then track promoted info if needed.
1.1285 + // When tracking (see PromotionInfo::track()), the mark word may
1.1286 + // be displaced and in this case restoration of the mark word
1.1287 + // occurs in the (oop_since_save_marks_)iterate phase.
1.1288 + if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {
1.1289 + // Out of space for allocating spooling buffers;
1.1290 + // try expanding and allocating spooling buffers.
1.1291 + if (!expand_and_ensure_spooling_space(promoInfo)) {
1.1292 + return NULL;
1.1293 + }
1.1294 + }
1.1295 + assert(promoInfo->has_spooling_space(), "Control point invariant");
1.1296 + HeapWord* obj_ptr = ps->lab.alloc(word_sz);
1.1297 + if (obj_ptr == NULL) {
1.1298 + obj_ptr = expand_and_par_lab_allocate(ps, word_sz);
1.1299 + if (obj_ptr == NULL) {
1.1300 + return NULL;
1.1301 + }
1.1302 + }
1.1303 + oop obj = oop(obj_ptr);
1.1304 + assert(obj->klass() == NULL, "Object should be uninitialized here.");
1.1305 + // Otherwise, copy the object. Here we must be careful to insert the
1.1306 + // klass pointer last, since this marks the block as an allocated object.
1.1307 + HeapWord* old_ptr = (HeapWord*)old;
1.1308 + if (word_sz > (size_t)oopDesc::header_size()) {
1.1309 + Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),
1.1310 + obj_ptr + oopDesc::header_size(),
1.1311 + word_sz - oopDesc::header_size());
1.1312 + }
1.1313 + // Restore the mark word copied above.
1.1314 + obj->set_mark(m);
1.1315 + // Now we can track the promoted object, if necessary. We take care
1.1316 + // To delay the transition from uninitialized to full object
1.1317 + // (i.e., insertion of klass pointer) until after, so that it
1.1318 + // atomically becomes a promoted object.
1.1319 + if (promoInfo->tracking()) {
1.1320 + promoInfo->track((PromotedObject*)obj, old->klass());
1.1321 + }
1.1322 + // Finally, install the klass pointer.
1.1323 + obj->set_klass(old->klass());
1.1324 +
1.1325 + assert(old->is_oop(), "Will dereference klass ptr below");
1.1326 + collector()->promoted(true, // parallel
1.1327 + obj_ptr, old->is_objArray(), word_sz);
1.1328 +
1.1329 + NOT_PRODUCT(
1.1330 + Atomic::inc(&_numObjectsPromoted);
1.1331 + Atomic::add((jint)CompactibleFreeListSpace::adjustObjectSize(obj->size()),
1.1332 + &_numWordsPromoted);
1.1333 + )
1.1334 +
1.1335 + return obj;
1.1336 +}
1.1337 +
1.1338 +void
1.1339 +ConcurrentMarkSweepGeneration::
1.1340 +par_promote_alloc_undo(int thread_num,
1.1341 + HeapWord* obj, size_t word_sz) {
1.1342 + // CMS does not support promotion undo.
1.1343 + ShouldNotReachHere();
1.1344 +}
1.1345 +
1.1346 +void
1.1347 +ConcurrentMarkSweepGeneration::
1.1348 +par_promote_alloc_done(int thread_num) {
1.1349 + CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1.1350 + ps->lab.retire();
1.1351 +#if CFLS_LAB_REFILL_STATS
1.1352 + if (thread_num == 0) {
1.1353 + _cmsSpace->print_par_alloc_stats();
1.1354 + }
1.1355 +#endif
1.1356 +}
1.1357 +
1.1358 +void
1.1359 +ConcurrentMarkSweepGeneration::
1.1360 +par_oop_since_save_marks_iterate_done(int thread_num) {
1.1361 + CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1.1362 + ParScanWithoutBarrierClosure* dummy_cl = NULL;
1.1363 + ps->promo.promoted_oops_iterate_nv(dummy_cl);
1.1364 +}
1.1365 +
1.1366 +// XXXPERM
1.1367 +bool ConcurrentMarkSweepGeneration::should_collect(bool full,
1.1368 + size_t size,
1.1369 + bool tlab)
1.1370 +{
1.1371 + // We allow a STW collection only if a full
1.1372 + // collection was requested.
1.1373 + return full || should_allocate(size, tlab); // FIX ME !!!
1.1374 + // This and promotion failure handling are connected at the
1.1375 + // hip and should be fixed by untying them.
1.1376 +}
1.1377 +
1.1378 +bool CMSCollector::shouldConcurrentCollect() {
1.1379 + if (_full_gc_requested) {
1.1380 + assert(ExplicitGCInvokesConcurrent, "Unexpected state");
1.1381 + if (Verbose && PrintGCDetails) {
1.1382 + gclog_or_tty->print_cr("CMSCollector: collect because of explicit "
1.1383 + " gc request");
1.1384 + }
1.1385 + return true;
1.1386 + }
1.1387 +
1.1388 + // For debugging purposes, change the type of collection.
1.1389 + // If the rotation is not on the concurrent collection
1.1390 + // type, don't start a concurrent collection.
1.1391 + NOT_PRODUCT(
1.1392 + if (RotateCMSCollectionTypes &&
1.1393 + (_cmsGen->debug_collection_type() !=
1.1394 + ConcurrentMarkSweepGeneration::Concurrent_collection_type)) {
1.1395 + assert(_cmsGen->debug_collection_type() !=
1.1396 + ConcurrentMarkSweepGeneration::Unknown_collection_type,
1.1397 + "Bad cms collection type");
1.1398 + return false;
1.1399 + }
1.1400 + )
1.1401 +
1.1402 + FreelistLocker x(this);
1.1403 + // ------------------------------------------------------------------
1.1404 + // Print out lots of information which affects the initiation of
1.1405 + // a collection.
1.1406 + if (PrintCMSInitiationStatistics && stats().valid()) {
1.1407 + gclog_or_tty->print("CMSCollector shouldConcurrentCollect: ");
1.1408 + gclog_or_tty->stamp();
1.1409 + gclog_or_tty->print_cr("");
1.1410 + stats().print_on(gclog_or_tty);
1.1411 + gclog_or_tty->print_cr("time_until_cms_gen_full %3.7f",
1.1412 + stats().time_until_cms_gen_full());
1.1413 + gclog_or_tty->print_cr("free="SIZE_FORMAT, _cmsGen->free());
1.1414 + gclog_or_tty->print_cr("contiguous_available="SIZE_FORMAT,
1.1415 + _cmsGen->contiguous_available());
1.1416 + gclog_or_tty->print_cr("promotion_rate=%g", stats().promotion_rate());
1.1417 + gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate());
1.1418 + gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy());
1.1419 + gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", initiatingOccupancy());
1.1420 + }
1.1421 + // ------------------------------------------------------------------
1.1422 +
1.1423 + // If the estimated time to complete a cms collection (cms_duration())
1.1424 + // is less than the estimated time remaining until the cms generation
1.1425 + // is full, start a collection.
1.1426 + if (!UseCMSInitiatingOccupancyOnly) {
1.1427 + if (stats().valid()) {
1.1428 + if (stats().time_until_cms_start() == 0.0) {
1.1429 + return true;
1.1430 + }
1.1431 + } else {
1.1432 + // We want to conservatively collect somewhat early in order
1.1433 + // to try and "bootstrap" our CMS/promotion statistics;
1.1434 + // this branch will not fire after the first successful CMS
1.1435 + // collection because the stats should then be valid.
1.1436 + if (_cmsGen->occupancy() >= _bootstrap_occupancy) {
1.1437 + if (Verbose && PrintGCDetails) {
1.1438 + gclog_or_tty->print_cr(
1.1439 + " CMSCollector: collect for bootstrapping statistics:"
1.1440 + " occupancy = %f, boot occupancy = %f", _cmsGen->occupancy(),
1.1441 + _bootstrap_occupancy);
1.1442 + }
1.1443 + return true;
1.1444 + }
1.1445 + }
1.1446 + }
1.1447 +
1.1448 + // Otherwise, we start a collection cycle if either the perm gen or
1.1449 + // old gen want a collection cycle started. Each may use
1.1450 + // an appropriate criterion for making this decision.
1.1451 + // XXX We need to make sure that the gen expansion
1.1452 + // criterion dovetails well with this.
1.1453 + if (_cmsGen->shouldConcurrentCollect(initiatingOccupancy())) {
1.1454 + if (Verbose && PrintGCDetails) {
1.1455 + gclog_or_tty->print_cr("CMS old gen initiated");
1.1456 + }
1.1457 + return true;
1.1458 + }
1.1459 +
1.1460 + if (cms_should_unload_classes() &&
1.1461 + _permGen->shouldConcurrentCollect(initiatingOccupancy())) {
1.1462 + if (Verbose && PrintGCDetails) {
1.1463 + gclog_or_tty->print_cr("CMS perm gen initiated");
1.1464 + }
1.1465 + return true;
1.1466 + }
1.1467 +
1.1468 + return false;
1.1469 +}
1.1470 +
1.1471 +// Clear _expansion_cause fields of constituent generations
1.1472 +void CMSCollector::clear_expansion_cause() {
1.1473 + _cmsGen->clear_expansion_cause();
1.1474 + _permGen->clear_expansion_cause();
1.1475 +}
1.1476 +
1.1477 +bool ConcurrentMarkSweepGeneration::shouldConcurrentCollect(
1.1478 + double initiatingOccupancy) {
1.1479 + // We should be conservative in starting a collection cycle. To
1.1480 + // start too eagerly runs the risk of collecting too often in the
1.1481 + // extreme. To collect too rarely falls back on full collections,
1.1482 + // which works, even if not optimum in terms of concurrent work.
1.1483 + // As a work around for too eagerly collecting, use the flag
1.1484 + // UseCMSInitiatingOccupancyOnly. This also has the advantage of
1.1485 + // giving the user an easily understandable way of controlling the
1.1486 + // collections.
1.1487 + // We want to start a new collection cycle if any of the following
1.1488 + // conditions hold:
1.1489 + // . our current occupancy exceeds the initiating occupancy, or
1.1490 + // . we recently needed to expand and have not since that expansion,
1.1491 + // collected, or
1.1492 + // . we are not using adaptive free lists and linear allocation is
1.1493 + // going to fail, or
1.1494 + // . (for old gen) incremental collection has already failed or
1.1495 + // may soon fail in the near future as we may not be able to absorb
1.1496 + // promotions.
1.1497 + assert_lock_strong(freelistLock());
1.1498 +
1.1499 + if (occupancy() > initiatingOccupancy) {
1.1500 + if (PrintGCDetails && Verbose) {
1.1501 + gclog_or_tty->print(" %s: collect because of occupancy %f / %f ",
1.1502 + short_name(), occupancy(), initiatingOccupancy);
1.1503 + }
1.1504 + return true;
1.1505 + }
1.1506 + if (UseCMSInitiatingOccupancyOnly) {
1.1507 + return false;
1.1508 + }
1.1509 + if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) {
1.1510 + if (PrintGCDetails && Verbose) {
1.1511 + gclog_or_tty->print(" %s: collect because expanded for allocation ",
1.1512 + short_name());
1.1513 + }
1.1514 + return true;
1.1515 + }
1.1516 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.1517 + assert(gch->collector_policy()->is_two_generation_policy(),
1.1518 + "You may want to check the correctness of the following");
1.1519 + if (gch->incremental_collection_will_fail()) {
1.1520 + if (PrintGCDetails && Verbose) {
1.1521 + gclog_or_tty->print(" %s: collect because incremental collection will fail ",
1.1522 + short_name());
1.1523 + }
1.1524 + return true;
1.1525 + }
1.1526 + if (!_cmsSpace->adaptive_freelists() &&
1.1527 + _cmsSpace->linearAllocationWouldFail()) {
1.1528 + if (PrintGCDetails && Verbose) {
1.1529 + gclog_or_tty->print(" %s: collect because of linAB ",
1.1530 + short_name());
1.1531 + }
1.1532 + return true;
1.1533 + }
1.1534 + return false;
1.1535 +}
1.1536 +
1.1537 +void ConcurrentMarkSweepGeneration::collect(bool full,
1.1538 + bool clear_all_soft_refs,
1.1539 + size_t size,
1.1540 + bool tlab)
1.1541 +{
1.1542 + collector()->collect(full, clear_all_soft_refs, size, tlab);
1.1543 +}
1.1544 +
1.1545 +void CMSCollector::collect(bool full,
1.1546 + bool clear_all_soft_refs,
1.1547 + size_t size,
1.1548 + bool tlab)
1.1549 +{
1.1550 + if (!UseCMSCollectionPassing && _collectorState > Idling) {
1.1551 + // For debugging purposes skip the collection if the state
1.1552 + // is not currently idle
1.1553 + if (TraceCMSState) {
1.1554 + gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " skipped full:%d CMS state %d",
1.1555 + Thread::current(), full, _collectorState);
1.1556 + }
1.1557 + return;
1.1558 + }
1.1559 +
1.1560 + // The following "if" branch is present for defensive reasons.
1.1561 + // In the current uses of this interface, it can be replaced with:
1.1562 + // assert(!GC_locker.is_active(), "Can't be called otherwise");
1.1563 + // But I am not placing that assert here to allow future
1.1564 + // generality in invoking this interface.
1.1565 + if (GC_locker::is_active()) {
1.1566 + // A consistency test for GC_locker
1.1567 + assert(GC_locker::needs_gc(), "Should have been set already");
1.1568 + // Skip this foreground collection, instead
1.1569 + // expanding the heap if necessary.
1.1570 + // Need the free list locks for the call to free() in compute_new_size()
1.1571 + compute_new_size();
1.1572 + return;
1.1573 + }
1.1574 + acquire_control_and_collect(full, clear_all_soft_refs);
1.1575 + _full_gcs_since_conc_gc++;
1.1576 +
1.1577 +}
1.1578 +
1.1579 +void CMSCollector::request_full_gc(unsigned int full_gc_count) {
1.1580 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.1581 + unsigned int gc_count = gch->total_full_collections();
1.1582 + if (gc_count == full_gc_count) {
1.1583 + MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag);
1.1584 + _full_gc_requested = true;
1.1585 + CGC_lock->notify(); // nudge CMS thread
1.1586 + }
1.1587 +}
1.1588 +
1.1589 +
1.1590 +// The foreground and background collectors need to coordinate in order
1.1591 +// to make sure that they do not mutually interfere with CMS collections.
1.1592 +// When a background collection is active,
1.1593 +// the foreground collector may need to take over (preempt) and
1.1594 +// synchronously complete an ongoing collection. Depending on the
1.1595 +// frequency of the background collections and the heap usage
1.1596 +// of the application, this preemption can be seldom or frequent.
1.1597 +// There are only certain
1.1598 +// points in the background collection that the "collection-baton"
1.1599 +// can be passed to the foreground collector.
1.1600 +//
1.1601 +// The foreground collector will wait for the baton before
1.1602 +// starting any part of the collection. The foreground collector
1.1603 +// will only wait at one location.
1.1604 +//
1.1605 +// The background collector will yield the baton before starting a new
1.1606 +// phase of the collection (e.g., before initial marking, marking from roots,
1.1607 +// precleaning, final re-mark, sweep etc.) This is normally done at the head
1.1608 +// of the loop which switches the phases. The background collector does some
1.1609 +// of the phases (initial mark, final re-mark) with the world stopped.
1.1610 +// Because of locking involved in stopping the world,
1.1611 +// the foreground collector should not block waiting for the background
1.1612 +// collector when it is doing a stop-the-world phase. The background
1.1613 +// collector will yield the baton at an additional point just before
1.1614 +// it enters a stop-the-world phase. Once the world is stopped, the
1.1615 +// background collector checks the phase of the collection. If the
1.1616 +// phase has not changed, it proceeds with the collection. If the
1.1617 +// phase has changed, it skips that phase of the collection. See
1.1618 +// the comments on the use of the Heap_lock in collect_in_background().
1.1619 +//
1.1620 +// Variable used in baton passing.
1.1621 +// _foregroundGCIsActive - Set to true by the foreground collector when
1.1622 +// it wants the baton. The foreground clears it when it has finished
1.1623 +// the collection.
1.1624 +// _foregroundGCShouldWait - Set to true by the background collector
1.1625 +// when it is running. The foreground collector waits while
1.1626 +// _foregroundGCShouldWait is true.
1.1627 +// CGC_lock - monitor used to protect access to the above variables
1.1628 +// and to notify the foreground and background collectors.
1.1629 +// _collectorState - current state of the CMS collection.
1.1630 +//
1.1631 +// The foreground collector
1.1632 +// acquires the CGC_lock
1.1633 +// sets _foregroundGCIsActive
1.1634 +// waits on the CGC_lock for _foregroundGCShouldWait to be false
1.1635 +// various locks acquired in preparation for the collection
1.1636 +// are released so as not to block the background collector
1.1637 +// that is in the midst of a collection
1.1638 +// proceeds with the collection
1.1639 +// clears _foregroundGCIsActive
1.1640 +// returns
1.1641 +//
1.1642 +// The background collector in a loop iterating on the phases of the
1.1643 +// collection
1.1644 +// acquires the CGC_lock
1.1645 +// sets _foregroundGCShouldWait
1.1646 +// if _foregroundGCIsActive is set
1.1647 +// clears _foregroundGCShouldWait, notifies _CGC_lock
1.1648 +// waits on _CGC_lock for _foregroundGCIsActive to become false
1.1649 +// and exits the loop.
1.1650 +// otherwise
1.1651 +// proceed with that phase of the collection
1.1652 +// if the phase is a stop-the-world phase,
1.1653 +// yield the baton once more just before enqueueing
1.1654 +// the stop-world CMS operation (executed by the VM thread).
1.1655 +// returns after all phases of the collection are done
1.1656 +//
1.1657 +
1.1658 +void CMSCollector::acquire_control_and_collect(bool full,
1.1659 + bool clear_all_soft_refs) {
1.1660 + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
1.1661 + assert(!Thread::current()->is_ConcurrentGC_thread(),
1.1662 + "shouldn't try to acquire control from self!");
1.1663 +
1.1664 + // Start the protocol for acquiring control of the
1.1665 + // collection from the background collector (aka CMS thread).
1.1666 + assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1.1667 + "VM thread should have CMS token");
1.1668 + // Remember the possibly interrupted state of an ongoing
1.1669 + // concurrent collection
1.1670 + CollectorState first_state = _collectorState;
1.1671 +
1.1672 + // Signal to a possibly ongoing concurrent collection that
1.1673 + // we want to do a foreground collection.
1.1674 + _foregroundGCIsActive = true;
1.1675 +
1.1676 + // Disable incremental mode during a foreground collection.
1.1677 + ICMSDisabler icms_disabler;
1.1678 +
1.1679 + // release locks and wait for a notify from the background collector
1.1680 + // releasing the locks in only necessary for phases which
1.1681 + // do yields to improve the granularity of the collection.
1.1682 + assert_lock_strong(bitMapLock());
1.1683 + // We need to lock the Free list lock for the space that we are
1.1684 + // currently collecting.
1.1685 + assert(haveFreelistLocks(), "Must be holding free list locks");
1.1686 + bitMapLock()->unlock();
1.1687 + releaseFreelistLocks();
1.1688 + {
1.1689 + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1.1690 + if (_foregroundGCShouldWait) {
1.1691 + // We are going to be waiting for action for the CMS thread;
1.1692 + // it had better not be gone (for instance at shutdown)!
1.1693 + assert(ConcurrentMarkSweepThread::cmst() != NULL,
1.1694 + "CMS thread must be running");
1.1695 + // Wait here until the background collector gives us the go-ahead
1.1696 + ConcurrentMarkSweepThread::clear_CMS_flag(
1.1697 + ConcurrentMarkSweepThread::CMS_vm_has_token); // release token
1.1698 + // Get a possibly blocked CMS thread going:
1.1699 + // Note that we set _foregroundGCIsActive true above,
1.1700 + // without protection of the CGC_lock.
1.1701 + CGC_lock->notify();
1.1702 + assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(),
1.1703 + "Possible deadlock");
1.1704 + while (_foregroundGCShouldWait) {
1.1705 + // wait for notification
1.1706 + CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1.1707 + // Possibility of delay/starvation here, since CMS token does
1.1708 + // not know to give priority to VM thread? Actually, i think
1.1709 + // there wouldn't be any delay/starvation, but the proof of
1.1710 + // that "fact" (?) appears non-trivial. XXX 20011219YSR
1.1711 + }
1.1712 + ConcurrentMarkSweepThread::set_CMS_flag(
1.1713 + ConcurrentMarkSweepThread::CMS_vm_has_token);
1.1714 + }
1.1715 + }
1.1716 + // The CMS_token is already held. Get back the other locks.
1.1717 + assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1.1718 + "VM thread should have CMS token");
1.1719 + getFreelistLocks();
1.1720 + bitMapLock()->lock_without_safepoint_check();
1.1721 + if (TraceCMSState) {
1.1722 + gclog_or_tty->print_cr("CMS foreground collector has asked for control "
1.1723 + INTPTR_FORMAT " with first state %d", Thread::current(), first_state);
1.1724 + gclog_or_tty->print_cr(" gets control with state %d", _collectorState);
1.1725 + }
1.1726 +
1.1727 + // Check if we need to do a compaction, or if not, whether
1.1728 + // we need to start the mark-sweep from scratch.
1.1729 + bool should_compact = false;
1.1730 + bool should_start_over = false;
1.1731 + decide_foreground_collection_type(clear_all_soft_refs,
1.1732 + &should_compact, &should_start_over);
1.1733 +
1.1734 +NOT_PRODUCT(
1.1735 + if (RotateCMSCollectionTypes) {
1.1736 + if (_cmsGen->debug_collection_type() ==
1.1737 + ConcurrentMarkSweepGeneration::MSC_foreground_collection_type) {
1.1738 + should_compact = true;
1.1739 + } else if (_cmsGen->debug_collection_type() ==
1.1740 + ConcurrentMarkSweepGeneration::MS_foreground_collection_type) {
1.1741 + should_compact = false;
1.1742 + }
1.1743 + }
1.1744 +)
1.1745 +
1.1746 + if (PrintGCDetails && first_state > Idling) {
1.1747 + GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause();
1.1748 + if (GCCause::is_user_requested_gc(cause) ||
1.1749 + GCCause::is_serviceability_requested_gc(cause)) {
1.1750 + gclog_or_tty->print(" (concurrent mode interrupted)");
1.1751 + } else {
1.1752 + gclog_or_tty->print(" (concurrent mode failure)");
1.1753 + }
1.1754 + }
1.1755 +
1.1756 + if (should_compact) {
1.1757 + // If the collection is being acquired from the background
1.1758 + // collector, there may be references on the discovered
1.1759 + // references lists that have NULL referents (being those
1.1760 + // that were concurrently cleared by a mutator) or
1.1761 + // that are no longer active (having been enqueued concurrently
1.1762 + // by the mutator).
1.1763 + // Scrub the list of those references because Mark-Sweep-Compact
1.1764 + // code assumes referents are not NULL and that all discovered
1.1765 + // Reference objects are active.
1.1766 + ref_processor()->clean_up_discovered_references();
1.1767 +
1.1768 + do_compaction_work(clear_all_soft_refs);
1.1769 +
1.1770 + // Has the GC time limit been exceeded?
1.1771 + check_gc_time_limit();
1.1772 +
1.1773 + } else {
1.1774 + do_mark_sweep_work(clear_all_soft_refs, first_state,
1.1775 + should_start_over);
1.1776 + }
1.1777 + // Reset the expansion cause, now that we just completed
1.1778 + // a collection cycle.
1.1779 + clear_expansion_cause();
1.1780 + _foregroundGCIsActive = false;
1.1781 + return;
1.1782 +}
1.1783 +
1.1784 +void CMSCollector::check_gc_time_limit() {
1.1785 +
1.1786 + // Ignore explicit GC's. Exiting here does not set the flag and
1.1787 + // does not reset the count. Updating of the averages for system
1.1788 + // GC's is still controlled by UseAdaptiveSizePolicyWithSystemGC.
1.1789 + GCCause::Cause gc_cause = GenCollectedHeap::heap()->gc_cause();
1.1790 + if (GCCause::is_user_requested_gc(gc_cause) ||
1.1791 + GCCause::is_serviceability_requested_gc(gc_cause)) {
1.1792 + return;
1.1793 + }
1.1794 +
1.1795 + // Calculate the fraction of the CMS generation was freed during
1.1796 + // the last collection.
1.1797 + // Only consider the STW compacting cost for now.
1.1798 + //
1.1799 + // Note that the gc time limit test only works for the collections
1.1800 + // of the young gen + tenured gen and not for collections of the
1.1801 + // permanent gen. That is because the calculation of the space
1.1802 + // freed by the collection is the free space in the young gen +
1.1803 + // tenured gen.
1.1804 +
1.1805 + double fraction_free =
1.1806 + ((double)_cmsGen->free())/((double)_cmsGen->max_capacity());
1.1807 + if ((100.0 * size_policy()->compacting_gc_cost()) >
1.1808 + ((double) GCTimeLimit) &&
1.1809 + ((fraction_free * 100) < GCHeapFreeLimit)) {
1.1810 + size_policy()->inc_gc_time_limit_count();
1.1811 + if (UseGCOverheadLimit &&
1.1812 + (size_policy()->gc_time_limit_count() >
1.1813 + AdaptiveSizePolicyGCTimeLimitThreshold)) {
1.1814 + size_policy()->set_gc_time_limit_exceeded(true);
1.1815 + // Avoid consecutive OOM due to the gc time limit by resetting
1.1816 + // the counter.
1.1817 + size_policy()->reset_gc_time_limit_count();
1.1818 + if (PrintGCDetails) {
1.1819 + gclog_or_tty->print_cr(" GC is exceeding overhead limit "
1.1820 + "of %d%%", GCTimeLimit);
1.1821 + }
1.1822 + } else {
1.1823 + if (PrintGCDetails) {
1.1824 + gclog_or_tty->print_cr(" GC would exceed overhead limit "
1.1825 + "of %d%%", GCTimeLimit);
1.1826 + }
1.1827 + }
1.1828 + } else {
1.1829 + size_policy()->reset_gc_time_limit_count();
1.1830 + }
1.1831 +}
1.1832 +
1.1833 +// Resize the perm generation and the tenured generation
1.1834 +// after obtaining the free list locks for the
1.1835 +// two generations.
1.1836 +void CMSCollector::compute_new_size() {
1.1837 + assert_locked_or_safepoint(Heap_lock);
1.1838 + FreelistLocker z(this);
1.1839 + _permGen->compute_new_size();
1.1840 + _cmsGen->compute_new_size();
1.1841 +}
1.1842 +
1.1843 +// A work method used by foreground collection to determine
1.1844 +// what type of collection (compacting or not, continuing or fresh)
1.1845 +// it should do.
1.1846 +// NOTE: the intent is to make UseCMSCompactAtFullCollection
1.1847 +// and CMSCompactWhenClearAllSoftRefs the default in the future
1.1848 +// and do away with the flags after a suitable period.
1.1849 +void CMSCollector::decide_foreground_collection_type(
1.1850 + bool clear_all_soft_refs, bool* should_compact,
1.1851 + bool* should_start_over) {
1.1852 + // Normally, we'll compact only if the UseCMSCompactAtFullCollection
1.1853 + // flag is set, and we have either requested a System.gc() or
1.1854 + // the number of full gc's since the last concurrent cycle
1.1855 + // has exceeded the threshold set by CMSFullGCsBeforeCompaction,
1.1856 + // or if an incremental collection has failed
1.1857 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.1858 + assert(gch->collector_policy()->is_two_generation_policy(),
1.1859 + "You may want to check the correctness of the following");
1.1860 + // Inform cms gen if this was due to partial collection failing.
1.1861 + // The CMS gen may use this fact to determine its expansion policy.
1.1862 + if (gch->incremental_collection_will_fail()) {
1.1863 + assert(!_cmsGen->incremental_collection_failed(),
1.1864 + "Should have been noticed, reacted to and cleared");
1.1865 + _cmsGen->set_incremental_collection_failed();
1.1866 + }
1.1867 + *should_compact =
1.1868 + UseCMSCompactAtFullCollection &&
1.1869 + ((_full_gcs_since_conc_gc >= CMSFullGCsBeforeCompaction) ||
1.1870 + GCCause::is_user_requested_gc(gch->gc_cause()) ||
1.1871 + gch->incremental_collection_will_fail());
1.1872 + *should_start_over = false;
1.1873 + if (clear_all_soft_refs && !*should_compact) {
1.1874 + // We are about to do a last ditch collection attempt
1.1875 + // so it would normally make sense to do a compaction
1.1876 + // to reclaim as much space as possible.
1.1877 + if (CMSCompactWhenClearAllSoftRefs) {
1.1878 + // Default: The rationale is that in this case either
1.1879 + // we are past the final marking phase, in which case
1.1880 + // we'd have to start over, or so little has been done
1.1881 + // that there's little point in saving that work. Compaction
1.1882 + // appears to be the sensible choice in either case.
1.1883 + *should_compact = true;
1.1884 + } else {
1.1885 + // We have been asked to clear all soft refs, but not to
1.1886 + // compact. Make sure that we aren't past the final checkpoint
1.1887 + // phase, for that is where we process soft refs. If we are already
1.1888 + // past that phase, we'll need to redo the refs discovery phase and
1.1889 + // if necessary clear soft refs that weren't previously
1.1890 + // cleared. We do so by remembering the phase in which
1.1891 + // we came in, and if we are past the refs processing
1.1892 + // phase, we'll choose to just redo the mark-sweep
1.1893 + // collection from scratch.
1.1894 + if (_collectorState > FinalMarking) {
1.1895 + // We are past the refs processing phase;
1.1896 + // start over and do a fresh synchronous CMS cycle
1.1897 + _collectorState = Resetting; // skip to reset to start new cycle
1.1898 + reset(false /* == !asynch */);
1.1899 + *should_start_over = true;
1.1900 + } // else we can continue a possibly ongoing current cycle
1.1901 + }
1.1902 + }
1.1903 +}
1.1904 +
1.1905 +// A work method used by the foreground collector to do
1.1906 +// a mark-sweep-compact.
1.1907 +void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
1.1908 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.1909 + TraceTime t("CMS:MSC ", PrintGCDetails && Verbose, true, gclog_or_tty);
1.1910 + if (PrintGC && Verbose && !(GCCause::is_user_requested_gc(gch->gc_cause()))) {
1.1911 + gclog_or_tty->print_cr("Compact ConcurrentMarkSweepGeneration after %d "
1.1912 + "collections passed to foreground collector", _full_gcs_since_conc_gc);
1.1913 + }
1.1914 +
1.1915 + // Sample collection interval time and reset for collection pause.
1.1916 + if (UseAdaptiveSizePolicy) {
1.1917 + size_policy()->msc_collection_begin();
1.1918 + }
1.1919 +
1.1920 + // Temporarily widen the span of the weak reference processing to
1.1921 + // the entire heap.
1.1922 + MemRegion new_span(GenCollectedHeap::heap()->reserved_region());
1.1923 + ReferenceProcessorSpanMutator x(ref_processor(), new_span);
1.1924 +
1.1925 + // Temporarily, clear the "is_alive_non_header" field of the
1.1926 + // reference processor.
1.1927 + ReferenceProcessorIsAliveMutator y(ref_processor(), NULL);
1.1928 +
1.1929 + // Temporarily make reference _processing_ single threaded (non-MT).
1.1930 + ReferenceProcessorMTProcMutator z(ref_processor(), false);
1.1931 +
1.1932 + // Temporarily make refs discovery atomic
1.1933 + ReferenceProcessorAtomicMutator w(ref_processor(), true);
1.1934 +
1.1935 + ref_processor()->set_enqueuing_is_done(false);
1.1936 + ref_processor()->enable_discovery();
1.1937 + // If an asynchronous collection finishes, the _modUnionTable is
1.1938 + // all clear. If we are assuming the collection from an asynchronous
1.1939 + // collection, clear the _modUnionTable.
1.1940 + assert(_collectorState != Idling || _modUnionTable.isAllClear(),
1.1941 + "_modUnionTable should be clear if the baton was not passed");
1.1942 + _modUnionTable.clear_all();
1.1943 +
1.1944 + // We must adjust the allocation statistics being maintained
1.1945 + // in the free list space. We do so by reading and clearing
1.1946 + // the sweep timer and updating the block flux rate estimates below.
1.1947 + assert(_sweep_timer.is_active(), "We should never see the timer inactive");
1.1948 + _sweep_timer.stop();
1.1949 + // Note that we do not use this sample to update the _sweep_estimate.
1.1950 + _cmsGen->cmsSpace()->beginSweepFLCensus((float)(_sweep_timer.seconds()),
1.1951 + _sweep_estimate.padded_average());
1.1952 +
1.1953 + GenMarkSweep::invoke_at_safepoint(_cmsGen->level(),
1.1954 + ref_processor(), clear_all_soft_refs);
1.1955 + #ifdef ASSERT
1.1956 + CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
1.1957 + size_t free_size = cms_space->free();
1.1958 + assert(free_size ==
1.1959 + pointer_delta(cms_space->end(), cms_space->compaction_top())
1.1960 + * HeapWordSize,
1.1961 + "All the free space should be compacted into one chunk at top");
1.1962 + assert(cms_space->dictionary()->totalChunkSize(
1.1963 + debug_only(cms_space->freelistLock())) == 0 ||
1.1964 + cms_space->totalSizeInIndexedFreeLists() == 0,
1.1965 + "All the free space should be in a single chunk");
1.1966 + size_t num = cms_space->totalCount();
1.1967 + assert((free_size == 0 && num == 0) ||
1.1968 + (free_size > 0 && (num == 1 || num == 2)),
1.1969 + "There should be at most 2 free chunks after compaction");
1.1970 + #endif // ASSERT
1.1971 + _collectorState = Resetting;
1.1972 + assert(_restart_addr == NULL,
1.1973 + "Should have been NULL'd before baton was passed");
1.1974 + reset(false /* == !asynch */);
1.1975 + _cmsGen->reset_after_compaction();
1.1976 +
1.1977 + if (verifying() && !cms_should_unload_classes()) {
1.1978 + perm_gen_verify_bit_map()->clear_all();
1.1979 + }
1.1980 +
1.1981 + // Clear any data recorded in the PLAB chunk arrays.
1.1982 + if (_survivor_plab_array != NULL) {
1.1983 + reset_survivor_plab_arrays();
1.1984 + }
1.1985 +
1.1986 + // Adjust the per-size allocation stats for the next epoch.
1.1987 + _cmsGen->cmsSpace()->endSweepFLCensus(sweepCount() /* fake */);
1.1988 + // Restart the "sweep timer" for next epoch.
1.1989 + _sweep_timer.reset();
1.1990 + _sweep_timer.start();
1.1991 +
1.1992 + // Sample collection pause time and reset for collection interval.
1.1993 + if (UseAdaptiveSizePolicy) {
1.1994 + size_policy()->msc_collection_end(gch->gc_cause());
1.1995 + }
1.1996 +
1.1997 + // For a mark-sweep-compact, compute_new_size() will be called
1.1998 + // in the heap's do_collection() method.
1.1999 +}
1.2000 +
1.2001 +// A work method used by the foreground collector to do
1.2002 +// a mark-sweep, after taking over from a possibly on-going
1.2003 +// concurrent mark-sweep collection.
1.2004 +void CMSCollector::do_mark_sweep_work(bool clear_all_soft_refs,
1.2005 + CollectorState first_state, bool should_start_over) {
1.2006 + if (PrintGC && Verbose) {
1.2007 + gclog_or_tty->print_cr("Pass concurrent collection to foreground "
1.2008 + "collector with count %d",
1.2009 + _full_gcs_since_conc_gc);
1.2010 + }
1.2011 + switch (_collectorState) {
1.2012 + case Idling:
1.2013 + if (first_state == Idling || should_start_over) {
1.2014 + // The background GC was not active, or should
1.2015 + // restarted from scratch; start the cycle.
1.2016 + _collectorState = InitialMarking;
1.2017 + }
1.2018 + // If first_state was not Idling, then a background GC
1.2019 + // was in progress and has now finished. No need to do it
1.2020 + // again. Leave the state as Idling.
1.2021 + break;
1.2022 + case Precleaning:
1.2023 + // In the foreground case don't do the precleaning since
1.2024 + // it is not done concurrently and there is extra work
1.2025 + // required.
1.2026 + _collectorState = FinalMarking;
1.2027 + }
1.2028 + if (PrintGCDetails &&
1.2029 + (_collectorState > Idling ||
1.2030 + !GCCause::is_user_requested_gc(GenCollectedHeap::heap()->gc_cause()))) {
1.2031 + gclog_or_tty->print(" (concurrent mode failure)");
1.2032 + }
1.2033 + collect_in_foreground(clear_all_soft_refs);
1.2034 +
1.2035 + // For a mark-sweep, compute_new_size() will be called
1.2036 + // in the heap's do_collection() method.
1.2037 +}
1.2038 +
1.2039 +
1.2040 +void CMSCollector::getFreelistLocks() const {
1.2041 + // Get locks for all free lists in all generations that this
1.2042 + // collector is responsible for
1.2043 + _cmsGen->freelistLock()->lock_without_safepoint_check();
1.2044 + _permGen->freelistLock()->lock_without_safepoint_check();
1.2045 +}
1.2046 +
1.2047 +void CMSCollector::releaseFreelistLocks() const {
1.2048 + // Release locks for all free lists in all generations that this
1.2049 + // collector is responsible for
1.2050 + _cmsGen->freelistLock()->unlock();
1.2051 + _permGen->freelistLock()->unlock();
1.2052 +}
1.2053 +
1.2054 +bool CMSCollector::haveFreelistLocks() const {
1.2055 + // Check locks for all free lists in all generations that this
1.2056 + // collector is responsible for
1.2057 + assert_lock_strong(_cmsGen->freelistLock());
1.2058 + assert_lock_strong(_permGen->freelistLock());
1.2059 + PRODUCT_ONLY(ShouldNotReachHere());
1.2060 + return true;
1.2061 +}
1.2062 +
1.2063 +// A utility class that is used by the CMS collector to
1.2064 +// temporarily "release" the foreground collector from its
1.2065 +// usual obligation to wait for the background collector to
1.2066 +// complete an ongoing phase before proceeding.
1.2067 +class ReleaseForegroundGC: public StackObj {
1.2068 + private:
1.2069 + CMSCollector* _c;
1.2070 + public:
1.2071 + ReleaseForegroundGC(CMSCollector* c) : _c(c) {
1.2072 + assert(_c->_foregroundGCShouldWait, "Else should not need to call");
1.2073 + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1.2074 + // allow a potentially blocked foreground collector to proceed
1.2075 + _c->_foregroundGCShouldWait = false;
1.2076 + if (_c->_foregroundGCIsActive) {
1.2077 + CGC_lock->notify();
1.2078 + }
1.2079 + assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.2080 + "Possible deadlock");
1.2081 + }
1.2082 +
1.2083 + ~ReleaseForegroundGC() {
1.2084 + assert(!_c->_foregroundGCShouldWait, "Usage protocol violation?");
1.2085 + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1.2086 + _c->_foregroundGCShouldWait = true;
1.2087 + }
1.2088 +};
1.2089 +
1.2090 +// There are separate collect_in_background and collect_in_foreground because of
1.2091 +// the different locking requirements of the background collector and the
1.2092 +// foreground collector. There was originally an attempt to share
1.2093 +// one "collect" method between the background collector and the foreground
1.2094 +// collector but the if-then-else required made it cleaner to have
1.2095 +// separate methods.
1.2096 +void CMSCollector::collect_in_background(bool clear_all_soft_refs) {
1.2097 + assert(Thread::current()->is_ConcurrentGC_thread(),
1.2098 + "A CMS asynchronous collection is only allowed on a CMS thread.");
1.2099 +
1.2100 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.2101 + {
1.2102 + bool safepoint_check = Mutex::_no_safepoint_check_flag;
1.2103 + MutexLockerEx hl(Heap_lock, safepoint_check);
1.2104 + MutexLockerEx x(CGC_lock, safepoint_check);
1.2105 + if (_foregroundGCIsActive || !UseAsyncConcMarkSweepGC) {
1.2106 + // The foreground collector is active or we're
1.2107 + // not using asynchronous collections. Skip this
1.2108 + // background collection.
1.2109 + assert(!_foregroundGCShouldWait, "Should be clear");
1.2110 + return;
1.2111 + } else {
1.2112 + assert(_collectorState == Idling, "Should be idling before start.");
1.2113 + _collectorState = InitialMarking;
1.2114 + // Reset the expansion cause, now that we are about to begin
1.2115 + // a new cycle.
1.2116 + clear_expansion_cause();
1.2117 + }
1.2118 + _unloaded_classes_last_cycle = cms_should_unload_classes(); // ... from last cycle
1.2119 + // This controls class unloading in response to an explicit gc request.
1.2120 + // If ExplicitGCInvokesConcurrentAndUnloadsClasses is set, then
1.2121 + // we will unload classes even if CMSClassUnloadingEnabled is not set.
1.2122 + // See CR 6541037 and related CRs.
1.2123 + _unload_classes = _full_gc_requested // ... for this cycle
1.2124 + && ExplicitGCInvokesConcurrentAndUnloadsClasses;
1.2125 + _full_gc_requested = false; // acks all outstanding full gc requests
1.2126 + // Signal that we are about to start a collection
1.2127 + gch->increment_total_full_collections(); // ... starting a collection cycle
1.2128 + _collection_count_start = gch->total_full_collections();
1.2129 + }
1.2130 +
1.2131 + // Used for PrintGC
1.2132 + size_t prev_used;
1.2133 + if (PrintGC && Verbose) {
1.2134 + prev_used = _cmsGen->used(); // XXXPERM
1.2135 + }
1.2136 +
1.2137 + // The change of the collection state is normally done at this level;
1.2138 + // the exceptions are phases that are executed while the world is
1.2139 + // stopped. For those phases the change of state is done while the
1.2140 + // world is stopped. For baton passing purposes this allows the
1.2141 + // background collector to finish the phase and change state atomically.
1.2142 + // The foreground collector cannot wait on a phase that is done
1.2143 + // while the world is stopped because the foreground collector already
1.2144 + // has the world stopped and would deadlock.
1.2145 + while (_collectorState != Idling) {
1.2146 + if (TraceCMSState) {
1.2147 + gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
1.2148 + Thread::current(), _collectorState);
1.2149 + }
1.2150 + // The foreground collector
1.2151 + // holds the Heap_lock throughout its collection.
1.2152 + // holds the CMS token (but not the lock)
1.2153 + // except while it is waiting for the background collector to yield.
1.2154 + //
1.2155 + // The foreground collector should be blocked (not for long)
1.2156 + // if the background collector is about to start a phase
1.2157 + // executed with world stopped. If the background
1.2158 + // collector has already started such a phase, the
1.2159 + // foreground collector is blocked waiting for the
1.2160 + // Heap_lock. The stop-world phases (InitialMarking and FinalMarking)
1.2161 + // are executed in the VM thread.
1.2162 + //
1.2163 + // The locking order is
1.2164 + // PendingListLock (PLL) -- if applicable (FinalMarking)
1.2165 + // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue())
1.2166 + // CMS token (claimed in
1.2167 + // stop_world_and_do() -->
1.2168 + // safepoint_synchronize() -->
1.2169 + // CMSThread::synchronize())
1.2170 +
1.2171 + {
1.2172 + // Check if the FG collector wants us to yield.
1.2173 + CMSTokenSync x(true); // is cms thread
1.2174 + if (waitForForegroundGC()) {
1.2175 + // We yielded to a foreground GC, nothing more to be
1.2176 + // done this round.
1.2177 + assert(_foregroundGCShouldWait == false, "We set it to false in "
1.2178 + "waitForForegroundGC()");
1.2179 + if (TraceCMSState) {
1.2180 + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1.2181 + " exiting collection CMS state %d",
1.2182 + Thread::current(), _collectorState);
1.2183 + }
1.2184 + return;
1.2185 + } else {
1.2186 + // The background collector can run but check to see if the
1.2187 + // foreground collector has done a collection while the
1.2188 + // background collector was waiting to get the CGC_lock
1.2189 + // above. If yes, break so that _foregroundGCShouldWait
1.2190 + // is cleared before returning.
1.2191 + if (_collectorState == Idling) {
1.2192 + break;
1.2193 + }
1.2194 + }
1.2195 + }
1.2196 +
1.2197 + assert(_foregroundGCShouldWait, "Foreground collector, if active, "
1.2198 + "should be waiting");
1.2199 +
1.2200 + switch (_collectorState) {
1.2201 + case InitialMarking:
1.2202 + {
1.2203 + ReleaseForegroundGC x(this);
1.2204 + stats().record_cms_begin();
1.2205 +
1.2206 + VM_CMS_Initial_Mark initial_mark_op(this);
1.2207 + VMThread::execute(&initial_mark_op);
1.2208 + }
1.2209 + // The collector state may be any legal state at this point
1.2210 + // since the background collector may have yielded to the
1.2211 + // foreground collector.
1.2212 + break;
1.2213 + case Marking:
1.2214 + // initial marking in checkpointRootsInitialWork has been completed
1.2215 + if (markFromRoots(true)) { // we were successful
1.2216 + assert(_collectorState == Precleaning, "Collector state should "
1.2217 + "have changed");
1.2218 + } else {
1.2219 + assert(_foregroundGCIsActive, "Internal state inconsistency");
1.2220 + }
1.2221 + break;
1.2222 + case Precleaning:
1.2223 + if (UseAdaptiveSizePolicy) {
1.2224 + size_policy()->concurrent_precleaning_begin();
1.2225 + }
1.2226 + // marking from roots in markFromRoots has been completed
1.2227 + preclean();
1.2228 + if (UseAdaptiveSizePolicy) {
1.2229 + size_policy()->concurrent_precleaning_end();
1.2230 + }
1.2231 + assert(_collectorState == AbortablePreclean ||
1.2232 + _collectorState == FinalMarking,
1.2233 + "Collector state should have changed");
1.2234 + break;
1.2235 + case AbortablePreclean:
1.2236 + if (UseAdaptiveSizePolicy) {
1.2237 + size_policy()->concurrent_phases_resume();
1.2238 + }
1.2239 + abortable_preclean();
1.2240 + if (UseAdaptiveSizePolicy) {
1.2241 + size_policy()->concurrent_precleaning_end();
1.2242 + }
1.2243 + assert(_collectorState == FinalMarking, "Collector state should "
1.2244 + "have changed");
1.2245 + break;
1.2246 + case FinalMarking:
1.2247 + {
1.2248 + ReleaseForegroundGC x(this);
1.2249 +
1.2250 + VM_CMS_Final_Remark final_remark_op(this);
1.2251 + VMThread::execute(&final_remark_op);
1.2252 + }
1.2253 + assert(_foregroundGCShouldWait, "block post-condition");
1.2254 + break;
1.2255 + case Sweeping:
1.2256 + if (UseAdaptiveSizePolicy) {
1.2257 + size_policy()->concurrent_sweeping_begin();
1.2258 + }
1.2259 + // final marking in checkpointRootsFinal has been completed
1.2260 + sweep(true);
1.2261 + assert(_collectorState == Resizing, "Collector state change "
1.2262 + "to Resizing must be done under the free_list_lock");
1.2263 + _full_gcs_since_conc_gc = 0;
1.2264 +
1.2265 + // Stop the timers for adaptive size policy for the concurrent phases
1.2266 + if (UseAdaptiveSizePolicy) {
1.2267 + size_policy()->concurrent_sweeping_end();
1.2268 + size_policy()->concurrent_phases_end(gch->gc_cause(),
1.2269 + gch->prev_gen(_cmsGen)->capacity(),
1.2270 + _cmsGen->free());
1.2271 + }
1.2272 +
1.2273 + case Resizing: {
1.2274 + // Sweeping has been completed...
1.2275 + // At this point the background collection has completed.
1.2276 + // Don't move the call to compute_new_size() down
1.2277 + // into code that might be executed if the background
1.2278 + // collection was preempted.
1.2279 + {
1.2280 + ReleaseForegroundGC x(this); // unblock FG collection
1.2281 + MutexLockerEx y(Heap_lock, Mutex::_no_safepoint_check_flag);
1.2282 + CMSTokenSync z(true); // not strictly needed.
1.2283 + if (_collectorState == Resizing) {
1.2284 + compute_new_size();
1.2285 + _collectorState = Resetting;
1.2286 + } else {
1.2287 + assert(_collectorState == Idling, "The state should only change"
1.2288 + " because the foreground collector has finished the collection");
1.2289 + }
1.2290 + }
1.2291 + break;
1.2292 + }
1.2293 + case Resetting:
1.2294 + // CMS heap resizing has been completed
1.2295 + reset(true);
1.2296 + assert(_collectorState == Idling, "Collector state should "
1.2297 + "have changed");
1.2298 + stats().record_cms_end();
1.2299 + // Don't move the concurrent_phases_end() and compute_new_size()
1.2300 + // calls to here because a preempted background collection
1.2301 + // has it's state set to "Resetting".
1.2302 + break;
1.2303 + case Idling:
1.2304 + default:
1.2305 + ShouldNotReachHere();
1.2306 + break;
1.2307 + }
1.2308 + if (TraceCMSState) {
1.2309 + gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
1.2310 + Thread::current(), _collectorState);
1.2311 + }
1.2312 + assert(_foregroundGCShouldWait, "block post-condition");
1.2313 + }
1.2314 +
1.2315 + // Should this be in gc_epilogue?
1.2316 + collector_policy()->counters()->update_counters();
1.2317 +
1.2318 + {
1.2319 + // Clear _foregroundGCShouldWait and, in the event that the
1.2320 + // foreground collector is waiting, notify it, before
1.2321 + // returning.
1.2322 + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1.2323 + _foregroundGCShouldWait = false;
1.2324 + if (_foregroundGCIsActive) {
1.2325 + CGC_lock->notify();
1.2326 + }
1.2327 + assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.2328 + "Possible deadlock");
1.2329 + }
1.2330 + if (TraceCMSState) {
1.2331 + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1.2332 + " exiting collection CMS state %d",
1.2333 + Thread::current(), _collectorState);
1.2334 + }
1.2335 + if (PrintGC && Verbose) {
1.2336 + _cmsGen->print_heap_change(prev_used);
1.2337 + }
1.2338 +}
1.2339 +
1.2340 +void CMSCollector::collect_in_foreground(bool clear_all_soft_refs) {
1.2341 + assert(_foregroundGCIsActive && !_foregroundGCShouldWait,
1.2342 + "Foreground collector should be waiting, not executing");
1.2343 + assert(Thread::current()->is_VM_thread(), "A foreground collection"
1.2344 + "may only be done by the VM Thread with the world stopped");
1.2345 + assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1.2346 + "VM thread should have CMS token");
1.2347 +
1.2348 + NOT_PRODUCT(TraceTime t("CMS:MS (foreground) ", PrintGCDetails && Verbose,
1.2349 + true, gclog_or_tty);)
1.2350 + if (UseAdaptiveSizePolicy) {
1.2351 + size_policy()->ms_collection_begin();
1.2352 + }
1.2353 + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact);
1.2354 +
1.2355 + HandleMark hm; // Discard invalid handles created during verification
1.2356 +
1.2357 + if (VerifyBeforeGC &&
1.2358 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2359 + Universe::verify(true);
1.2360 + }
1.2361 +
1.2362 + bool init_mark_was_synchronous = false; // until proven otherwise
1.2363 + while (_collectorState != Idling) {
1.2364 + if (TraceCMSState) {
1.2365 + gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
1.2366 + Thread::current(), _collectorState);
1.2367 + }
1.2368 + switch (_collectorState) {
1.2369 + case InitialMarking:
1.2370 + init_mark_was_synchronous = true; // fact to be exploited in re-mark
1.2371 + checkpointRootsInitial(false);
1.2372 + assert(_collectorState == Marking, "Collector state should have changed"
1.2373 + " within checkpointRootsInitial()");
1.2374 + break;
1.2375 + case Marking:
1.2376 + // initial marking in checkpointRootsInitialWork has been completed
1.2377 + if (VerifyDuringGC &&
1.2378 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2379 + gclog_or_tty->print("Verify before initial mark: ");
1.2380 + Universe::verify(true);
1.2381 + }
1.2382 + {
1.2383 + bool res = markFromRoots(false);
1.2384 + assert(res && _collectorState == FinalMarking, "Collector state should "
1.2385 + "have changed");
1.2386 + break;
1.2387 + }
1.2388 + case FinalMarking:
1.2389 + if (VerifyDuringGC &&
1.2390 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2391 + gclog_or_tty->print("Verify before re-mark: ");
1.2392 + Universe::verify(true);
1.2393 + }
1.2394 + checkpointRootsFinal(false, clear_all_soft_refs,
1.2395 + init_mark_was_synchronous);
1.2396 + assert(_collectorState == Sweeping, "Collector state should not "
1.2397 + "have changed within checkpointRootsFinal()");
1.2398 + break;
1.2399 + case Sweeping:
1.2400 + // final marking in checkpointRootsFinal has been completed
1.2401 + if (VerifyDuringGC &&
1.2402 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2403 + gclog_or_tty->print("Verify before sweep: ");
1.2404 + Universe::verify(true);
1.2405 + }
1.2406 + sweep(false);
1.2407 + assert(_collectorState == Resizing, "Incorrect state");
1.2408 + break;
1.2409 + case Resizing: {
1.2410 + // Sweeping has been completed; the actual resize in this case
1.2411 + // is done separately; nothing to be done in this state.
1.2412 + _collectorState = Resetting;
1.2413 + break;
1.2414 + }
1.2415 + case Resetting:
1.2416 + // The heap has been resized.
1.2417 + if (VerifyDuringGC &&
1.2418 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2419 + gclog_or_tty->print("Verify before reset: ");
1.2420 + Universe::verify(true);
1.2421 + }
1.2422 + reset(false);
1.2423 + assert(_collectorState == Idling, "Collector state should "
1.2424 + "have changed");
1.2425 + break;
1.2426 + case Precleaning:
1.2427 + case AbortablePreclean:
1.2428 + // Elide the preclean phase
1.2429 + _collectorState = FinalMarking;
1.2430 + break;
1.2431 + default:
1.2432 + ShouldNotReachHere();
1.2433 + }
1.2434 + if (TraceCMSState) {
1.2435 + gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
1.2436 + Thread::current(), _collectorState);
1.2437 + }
1.2438 + }
1.2439 +
1.2440 + if (UseAdaptiveSizePolicy) {
1.2441 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.2442 + size_policy()->ms_collection_end(gch->gc_cause());
1.2443 + }
1.2444 +
1.2445 + if (VerifyAfterGC &&
1.2446 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2447 + Universe::verify(true);
1.2448 + }
1.2449 + if (TraceCMSState) {
1.2450 + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1.2451 + " exiting collection CMS state %d",
1.2452 + Thread::current(), _collectorState);
1.2453 + }
1.2454 +}
1.2455 +
1.2456 +bool CMSCollector::waitForForegroundGC() {
1.2457 + bool res = false;
1.2458 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.2459 + "CMS thread should have CMS token");
1.2460 + // Block the foreground collector until the
1.2461 + // background collectors decides whether to
1.2462 + // yield.
1.2463 + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1.2464 + _foregroundGCShouldWait = true;
1.2465 + if (_foregroundGCIsActive) {
1.2466 + // The background collector yields to the
1.2467 + // foreground collector and returns a value
1.2468 + // indicating that it has yielded. The foreground
1.2469 + // collector can proceed.
1.2470 + res = true;
1.2471 + _foregroundGCShouldWait = false;
1.2472 + ConcurrentMarkSweepThread::clear_CMS_flag(
1.2473 + ConcurrentMarkSweepThread::CMS_cms_has_token);
1.2474 + ConcurrentMarkSweepThread::set_CMS_flag(
1.2475 + ConcurrentMarkSweepThread::CMS_cms_wants_token);
1.2476 + // Get a possibly blocked foreground thread going
1.2477 + CGC_lock->notify();
1.2478 + if (TraceCMSState) {
1.2479 + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d",
1.2480 + Thread::current(), _collectorState);
1.2481 + }
1.2482 + while (_foregroundGCIsActive) {
1.2483 + CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1.2484 + }
1.2485 + ConcurrentMarkSweepThread::set_CMS_flag(
1.2486 + ConcurrentMarkSweepThread::CMS_cms_has_token);
1.2487 + ConcurrentMarkSweepThread::clear_CMS_flag(
1.2488 + ConcurrentMarkSweepThread::CMS_cms_wants_token);
1.2489 + }
1.2490 + if (TraceCMSState) {
1.2491 + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d",
1.2492 + Thread::current(), _collectorState);
1.2493 + }
1.2494 + return res;
1.2495 +}
1.2496 +
1.2497 +// Because of the need to lock the free lists and other structures in
1.2498 +// the collector, common to all the generations that the collector is
1.2499 +// collecting, we need the gc_prologues of individual CMS generations
1.2500 +// delegate to their collector. It may have been simpler had the
1.2501 +// current infrastructure allowed one to call a prologue on a
1.2502 +// collector. In the absence of that we have the generation's
1.2503 +// prologue delegate to the collector, which delegates back
1.2504 +// some "local" work to a worker method in the individual generations
1.2505 +// that it's responsible for collecting, while itself doing any
1.2506 +// work common to all generations it's responsible for. A similar
1.2507 +// comment applies to the gc_epilogue()'s.
1.2508 +// The role of the varaible _between_prologue_and_epilogue is to
1.2509 +// enforce the invocation protocol.
1.2510 +void CMSCollector::gc_prologue(bool full) {
1.2511 + // Call gc_prologue_work() for each CMSGen and PermGen that
1.2512 + // we are responsible for.
1.2513 +
1.2514 + // The following locking discipline assumes that we are only called
1.2515 + // when the world is stopped.
1.2516 + assert(SafepointSynchronize::is_at_safepoint(), "world is stopped assumption");
1.2517 +
1.2518 + // The CMSCollector prologue must call the gc_prologues for the
1.2519 + // "generations" (including PermGen if any) that it's responsible
1.2520 + // for.
1.2521 +
1.2522 + assert( Thread::current()->is_VM_thread()
1.2523 + || ( CMSScavengeBeforeRemark
1.2524 + && Thread::current()->is_ConcurrentGC_thread()),
1.2525 + "Incorrect thread type for prologue execution");
1.2526 +
1.2527 + if (_between_prologue_and_epilogue) {
1.2528 + // We have already been invoked; this is a gc_prologue delegation
1.2529 + // from yet another CMS generation that we are responsible for, just
1.2530 + // ignore it since all relevant work has already been done.
1.2531 + return;
1.2532 + }
1.2533 +
1.2534 + // set a bit saying prologue has been called; cleared in epilogue
1.2535 + _between_prologue_and_epilogue = true;
1.2536 + // Claim locks for common data structures, then call gc_prologue_work()
1.2537 + // for each CMSGen and PermGen that we are responsible for.
1.2538 +
1.2539 + getFreelistLocks(); // gets free list locks on constituent spaces
1.2540 + bitMapLock()->lock_without_safepoint_check();
1.2541 +
1.2542 + // Should call gc_prologue_work() for all cms gens we are responsible for
1.2543 + bool registerClosure = _collectorState >= Marking
1.2544 + && _collectorState < Sweeping;
1.2545 + ModUnionClosure* muc = ParallelGCThreads > 0 ? &_modUnionClosurePar
1.2546 + : &_modUnionClosure;
1.2547 + _cmsGen->gc_prologue_work(full, registerClosure, muc);
1.2548 + _permGen->gc_prologue_work(full, registerClosure, muc);
1.2549 +
1.2550 + if (!full) {
1.2551 + stats().record_gc0_begin();
1.2552 + }
1.2553 +}
1.2554 +
1.2555 +void ConcurrentMarkSweepGeneration::gc_prologue(bool full) {
1.2556 + // Delegate to CMScollector which knows how to coordinate between
1.2557 + // this and any other CMS generations that it is responsible for
1.2558 + // collecting.
1.2559 + collector()->gc_prologue(full);
1.2560 +}
1.2561 +
1.2562 +// This is a "private" interface for use by this generation's CMSCollector.
1.2563 +// Not to be called directly by any other entity (for instance,
1.2564 +// GenCollectedHeap, which calls the "public" gc_prologue method above).
1.2565 +void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
1.2566 + bool registerClosure, ModUnionClosure* modUnionClosure) {
1.2567 + assert(!incremental_collection_failed(), "Shouldn't be set yet");
1.2568 + assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,
1.2569 + "Should be NULL");
1.2570 + if (registerClosure) {
1.2571 + cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure);
1.2572 + }
1.2573 + cmsSpace()->gc_prologue();
1.2574 + // Clear stat counters
1.2575 + NOT_PRODUCT(
1.2576 + assert(_numObjectsPromoted == 0, "check");
1.2577 + assert(_numWordsPromoted == 0, "check");
1.2578 + if (Verbose && PrintGC) {
1.2579 + gclog_or_tty->print("Allocated "SIZE_FORMAT" objects, "
1.2580 + SIZE_FORMAT" bytes concurrently",
1.2581 + _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord));
1.2582 + }
1.2583 + _numObjectsAllocated = 0;
1.2584 + _numWordsAllocated = 0;
1.2585 + )
1.2586 +}
1.2587 +
1.2588 +void CMSCollector::gc_epilogue(bool full) {
1.2589 + // The following locking discipline assumes that we are only called
1.2590 + // when the world is stopped.
1.2591 + assert(SafepointSynchronize::is_at_safepoint(),
1.2592 + "world is stopped assumption");
1.2593 +
1.2594 + // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks
1.2595 + // if linear allocation blocks need to be appropriately marked to allow the
1.2596 + // the blocks to be parsable. We also check here whether we need to nudge the
1.2597 + // CMS collector thread to start a new cycle (if it's not already active).
1.2598 + assert( Thread::current()->is_VM_thread()
1.2599 + || ( CMSScavengeBeforeRemark
1.2600 + && Thread::current()->is_ConcurrentGC_thread()),
1.2601 + "Incorrect thread type for epilogue execution");
1.2602 +
1.2603 + if (!_between_prologue_and_epilogue) {
1.2604 + // We have already been invoked; this is a gc_epilogue delegation
1.2605 + // from yet another CMS generation that we are responsible for, just
1.2606 + // ignore it since all relevant work has already been done.
1.2607 + return;
1.2608 + }
1.2609 + assert(haveFreelistLocks(), "must have freelist locks");
1.2610 + assert_lock_strong(bitMapLock());
1.2611 +
1.2612 + _cmsGen->gc_epilogue_work(full);
1.2613 + _permGen->gc_epilogue_work(full);
1.2614 +
1.2615 + if (_collectorState == AbortablePreclean || _collectorState == Precleaning) {
1.2616 + // in case sampling was not already enabled, enable it
1.2617 + _start_sampling = true;
1.2618 + }
1.2619 + // reset _eden_chunk_array so sampling starts afresh
1.2620 + _eden_chunk_index = 0;
1.2621 +
1.2622 + size_t cms_used = _cmsGen->cmsSpace()->used();
1.2623 + size_t perm_used = _permGen->cmsSpace()->used();
1.2624 +
1.2625 + // update performance counters - this uses a special version of
1.2626 + // update_counters() that allows the utilization to be passed as a
1.2627 + // parameter, avoiding multiple calls to used().
1.2628 + //
1.2629 + _cmsGen->update_counters(cms_used);
1.2630 + _permGen->update_counters(perm_used);
1.2631 +
1.2632 + if (CMSIncrementalMode) {
1.2633 + icms_update_allocation_limits();
1.2634 + }
1.2635 +
1.2636 + bitMapLock()->unlock();
1.2637 + releaseFreelistLocks();
1.2638 +
1.2639 + _between_prologue_and_epilogue = false; // ready for next cycle
1.2640 +}
1.2641 +
1.2642 +void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
1.2643 + collector()->gc_epilogue(full);
1.2644 +
1.2645 + // Also reset promotion tracking in par gc thread states.
1.2646 + if (ParallelGCThreads > 0) {
1.2647 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.2648 + _par_gc_thread_states[i]->promo.stopTrackingPromotions();
1.2649 + }
1.2650 + }
1.2651 +}
1.2652 +
1.2653 +void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) {
1.2654 + assert(!incremental_collection_failed(), "Should have been cleared");
1.2655 + cmsSpace()->setPreconsumptionDirtyCardClosure(NULL);
1.2656 + cmsSpace()->gc_epilogue();
1.2657 + // Print stat counters
1.2658 + NOT_PRODUCT(
1.2659 + assert(_numObjectsAllocated == 0, "check");
1.2660 + assert(_numWordsAllocated == 0, "check");
1.2661 + if (Verbose && PrintGC) {
1.2662 + gclog_or_tty->print("Promoted "SIZE_FORMAT" objects, "
1.2663 + SIZE_FORMAT" bytes",
1.2664 + _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord));
1.2665 + }
1.2666 + _numObjectsPromoted = 0;
1.2667 + _numWordsPromoted = 0;
1.2668 + )
1.2669 +
1.2670 + if (PrintGC && Verbose) {
1.2671 + // Call down the chain in contiguous_available needs the freelistLock
1.2672 + // so print this out before releasing the freeListLock.
1.2673 + gclog_or_tty->print(" Contiguous available "SIZE_FORMAT" bytes ",
1.2674 + contiguous_available());
1.2675 + }
1.2676 +}
1.2677 +
1.2678 +#ifndef PRODUCT
1.2679 +bool CMSCollector::have_cms_token() {
1.2680 + Thread* thr = Thread::current();
1.2681 + if (thr->is_VM_thread()) {
1.2682 + return ConcurrentMarkSweepThread::vm_thread_has_cms_token();
1.2683 + } else if (thr->is_ConcurrentGC_thread()) {
1.2684 + return ConcurrentMarkSweepThread::cms_thread_has_cms_token();
1.2685 + } else if (thr->is_GC_task_thread()) {
1.2686 + return ConcurrentMarkSweepThread::vm_thread_has_cms_token() &&
1.2687 + ParGCRareEvent_lock->owned_by_self();
1.2688 + }
1.2689 + return false;
1.2690 +}
1.2691 +#endif
1.2692 +
1.2693 +// Check reachability of the given heap address in CMS generation,
1.2694 +// treating all other generations as roots.
1.2695 +bool CMSCollector::is_cms_reachable(HeapWord* addr) {
1.2696 + // We could "guarantee" below, rather than assert, but i'll
1.2697 + // leave these as "asserts" so that an adventurous debugger
1.2698 + // could try this in the product build provided some subset of
1.2699 + // the conditions were met, provided they were intersted in the
1.2700 + // results and knew that the computation below wouldn't interfere
1.2701 + // with other concurrent computations mutating the structures
1.2702 + // being read or written.
1.2703 + assert(SafepointSynchronize::is_at_safepoint(),
1.2704 + "Else mutations in object graph will make answer suspect");
1.2705 + assert(have_cms_token(), "Should hold cms token");
1.2706 + assert(haveFreelistLocks(), "must hold free list locks");
1.2707 + assert_lock_strong(bitMapLock());
1.2708 +
1.2709 + // Clear the marking bit map array before starting, but, just
1.2710 + // for kicks, first report if the given address is already marked
1.2711 + gclog_or_tty->print_cr("Start: Address 0x%x is%s marked", addr,
1.2712 + _markBitMap.isMarked(addr) ? "" : " not");
1.2713 +
1.2714 + if (verify_after_remark()) {
1.2715 + MutexLockerEx x(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
1.2716 + bool result = verification_mark_bm()->isMarked(addr);
1.2717 + gclog_or_tty->print_cr("TransitiveMark: Address 0x%x %s marked", addr,
1.2718 + result ? "IS" : "is NOT");
1.2719 + return result;
1.2720 + } else {
1.2721 + gclog_or_tty->print_cr("Could not compute result");
1.2722 + return false;
1.2723 + }
1.2724 +}
1.2725 +
1.2726 +////////////////////////////////////////////////////////
1.2727 +// CMS Verification Support
1.2728 +////////////////////////////////////////////////////////
1.2729 +// Following the remark phase, the following invariant
1.2730 +// should hold -- each object in the CMS heap which is
1.2731 +// marked in markBitMap() should be marked in the verification_mark_bm().
1.2732 +
1.2733 +class VerifyMarkedClosure: public BitMapClosure {
1.2734 + CMSBitMap* _marks;
1.2735 + bool _failed;
1.2736 +
1.2737 + public:
1.2738 + VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
1.2739 +
1.2740 + void do_bit(size_t offset) {
1.2741 + HeapWord* addr = _marks->offsetToHeapWord(offset);
1.2742 + if (!_marks->isMarked(addr)) {
1.2743 + oop(addr)->print();
1.2744 + gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", addr);
1.2745 + _failed = true;
1.2746 + }
1.2747 + }
1.2748 +
1.2749 + bool failed() { return _failed; }
1.2750 +};
1.2751 +
1.2752 +bool CMSCollector::verify_after_remark() {
1.2753 + gclog_or_tty->print(" [Verifying CMS Marking... ");
1.2754 + MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
1.2755 + static bool init = false;
1.2756 +
1.2757 + assert(SafepointSynchronize::is_at_safepoint(),
1.2758 + "Else mutations in object graph will make answer suspect");
1.2759 + assert(have_cms_token(),
1.2760 + "Else there may be mutual interference in use of "
1.2761 + " verification data structures");
1.2762 + assert(_collectorState > Marking && _collectorState <= Sweeping,
1.2763 + "Else marking info checked here may be obsolete");
1.2764 + assert(haveFreelistLocks(), "must hold free list locks");
1.2765 + assert_lock_strong(bitMapLock());
1.2766 +
1.2767 +
1.2768 + // Allocate marking bit map if not already allocated
1.2769 + if (!init) { // first time
1.2770 + if (!verification_mark_bm()->allocate(_span)) {
1.2771 + return false;
1.2772 + }
1.2773 + init = true;
1.2774 + }
1.2775 +
1.2776 + assert(verification_mark_stack()->isEmpty(), "Should be empty");
1.2777 +
1.2778 + // Turn off refs discovery -- so we will be tracing through refs.
1.2779 + // This is as intended, because by this time
1.2780 + // GC must already have cleared any refs that need to be cleared,
1.2781 + // and traced those that need to be marked; moreover,
1.2782 + // the marking done here is not going to intefere in any
1.2783 + // way with the marking information used by GC.
1.2784 + NoRefDiscovery no_discovery(ref_processor());
1.2785 +
1.2786 + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
1.2787 +
1.2788 + // Clear any marks from a previous round
1.2789 + verification_mark_bm()->clear_all();
1.2790 + assert(verification_mark_stack()->isEmpty(), "markStack should be empty");
1.2791 + assert(overflow_list_is_empty(), "overflow list should be empty");
1.2792 +
1.2793 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.2794 + gch->ensure_parsability(false); // fill TLABs, but no need to retire them
1.2795 + // Update the saved marks which may affect the root scans.
1.2796 + gch->save_marks();
1.2797 +
1.2798 + if (CMSRemarkVerifyVariant == 1) {
1.2799 + // In this first variant of verification, we complete
1.2800 + // all marking, then check if the new marks-verctor is
1.2801 + // a subset of the CMS marks-vector.
1.2802 + verify_after_remark_work_1();
1.2803 + } else if (CMSRemarkVerifyVariant == 2) {
1.2804 + // In this second variant of verification, we flag an error
1.2805 + // (i.e. an object reachable in the new marks-vector not reachable
1.2806 + // in the CMS marks-vector) immediately, also indicating the
1.2807 + // identify of an object (A) that references the unmarked object (B) --
1.2808 + // presumably, a mutation to A failed to be picked up by preclean/remark?
1.2809 + verify_after_remark_work_2();
1.2810 + } else {
1.2811 + warning("Unrecognized value %d for CMSRemarkVerifyVariant",
1.2812 + CMSRemarkVerifyVariant);
1.2813 + }
1.2814 + gclog_or_tty->print(" done] ");
1.2815 + return true;
1.2816 +}
1.2817 +
1.2818 +void CMSCollector::verify_after_remark_work_1() {
1.2819 + ResourceMark rm;
1.2820 + HandleMark hm;
1.2821 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.2822 +
1.2823 + // Mark from roots one level into CMS
1.2824 + MarkRefsIntoClosure notOlder(_span, verification_mark_bm(), true /* nmethods */);
1.2825 + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
1.2826 +
1.2827 + gch->gen_process_strong_roots(_cmsGen->level(),
1.2828 + true, // younger gens are roots
1.2829 + true, // collecting perm gen
1.2830 + SharedHeap::ScanningOption(roots_scanning_options()),
1.2831 + NULL, ¬Older);
1.2832 +
1.2833 + // Now mark from the roots
1.2834 + assert(_revisitStack.isEmpty(), "Should be empty");
1.2835 + MarkFromRootsClosure markFromRootsClosure(this, _span,
1.2836 + verification_mark_bm(), verification_mark_stack(), &_revisitStack,
1.2837 + false /* don't yield */, true /* verifying */);
1.2838 + assert(_restart_addr == NULL, "Expected pre-condition");
1.2839 + verification_mark_bm()->iterate(&markFromRootsClosure);
1.2840 + while (_restart_addr != NULL) {
1.2841 + // Deal with stack overflow: by restarting at the indicated
1.2842 + // address.
1.2843 + HeapWord* ra = _restart_addr;
1.2844 + markFromRootsClosure.reset(ra);
1.2845 + _restart_addr = NULL;
1.2846 + verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
1.2847 + }
1.2848 + assert(verification_mark_stack()->isEmpty(), "Should have been drained");
1.2849 + verify_work_stacks_empty();
1.2850 + // Should reset the revisit stack above, since no class tree
1.2851 + // surgery is forthcoming.
1.2852 + _revisitStack.reset(); // throwing away all contents
1.2853 +
1.2854 + // Marking completed -- now verify that each bit marked in
1.2855 + // verification_mark_bm() is also marked in markBitMap(); flag all
1.2856 + // errors by printing corresponding objects.
1.2857 + VerifyMarkedClosure vcl(markBitMap());
1.2858 + verification_mark_bm()->iterate(&vcl);
1.2859 + if (vcl.failed()) {
1.2860 + gclog_or_tty->print("Verification failed");
1.2861 + Universe::heap()->print();
1.2862 + fatal(" ... aborting");
1.2863 + }
1.2864 +}
1.2865 +
1.2866 +void CMSCollector::verify_after_remark_work_2() {
1.2867 + ResourceMark rm;
1.2868 + HandleMark hm;
1.2869 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.2870 +
1.2871 + // Mark from roots one level into CMS
1.2872 + MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(),
1.2873 + markBitMap(), true /* nmethods */);
1.2874 + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
1.2875 + gch->gen_process_strong_roots(_cmsGen->level(),
1.2876 + true, // younger gens are roots
1.2877 + true, // collecting perm gen
1.2878 + SharedHeap::ScanningOption(roots_scanning_options()),
1.2879 + NULL, ¬Older);
1.2880 +
1.2881 + // Now mark from the roots
1.2882 + assert(_revisitStack.isEmpty(), "Should be empty");
1.2883 + MarkFromRootsVerifyClosure markFromRootsClosure(this, _span,
1.2884 + verification_mark_bm(), markBitMap(), verification_mark_stack());
1.2885 + assert(_restart_addr == NULL, "Expected pre-condition");
1.2886 + verification_mark_bm()->iterate(&markFromRootsClosure);
1.2887 + while (_restart_addr != NULL) {
1.2888 + // Deal with stack overflow: by restarting at the indicated
1.2889 + // address.
1.2890 + HeapWord* ra = _restart_addr;
1.2891 + markFromRootsClosure.reset(ra);
1.2892 + _restart_addr = NULL;
1.2893 + verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
1.2894 + }
1.2895 + assert(verification_mark_stack()->isEmpty(), "Should have been drained");
1.2896 + verify_work_stacks_empty();
1.2897 + // Should reset the revisit stack above, since no class tree
1.2898 + // surgery is forthcoming.
1.2899 + _revisitStack.reset(); // throwing away all contents
1.2900 +
1.2901 + // Marking completed -- now verify that each bit marked in
1.2902 + // verification_mark_bm() is also marked in markBitMap(); flag all
1.2903 + // errors by printing corresponding objects.
1.2904 + VerifyMarkedClosure vcl(markBitMap());
1.2905 + verification_mark_bm()->iterate(&vcl);
1.2906 + assert(!vcl.failed(), "Else verification above should not have succeeded");
1.2907 +}
1.2908 +
1.2909 +void ConcurrentMarkSweepGeneration::save_marks() {
1.2910 + // delegate to CMS space
1.2911 + cmsSpace()->save_marks();
1.2912 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.2913 + _par_gc_thread_states[i]->promo.startTrackingPromotions();
1.2914 + }
1.2915 +}
1.2916 +
1.2917 +bool ConcurrentMarkSweepGeneration::no_allocs_since_save_marks() {
1.2918 + return cmsSpace()->no_allocs_since_save_marks();
1.2919 +}
1.2920 +
1.2921 +#define CMS_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \
1.2922 + \
1.2923 +void ConcurrentMarkSweepGeneration:: \
1.2924 +oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \
1.2925 + cl->set_generation(this); \
1.2926 + cmsSpace()->oop_since_save_marks_iterate##nv_suffix(cl); \
1.2927 + cl->reset_generation(); \
1.2928 + save_marks(); \
1.2929 +}
1.2930 +
1.2931 +ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DEFN)
1.2932 +
1.2933 +void
1.2934 +ConcurrentMarkSweepGeneration::object_iterate_since_last_GC(ObjectClosure* blk)
1.2935 +{
1.2936 + // Not currently implemented; need to do the following. -- ysr.
1.2937 + // dld -- I think that is used for some sort of allocation profiler. So it
1.2938 + // really means the objects allocated by the mutator since the last
1.2939 + // GC. We could potentially implement this cheaply by recording only
1.2940 + // the direct allocations in a side data structure.
1.2941 + //
1.2942 + // I think we probably ought not to be required to support these
1.2943 + // iterations at any arbitrary point; I think there ought to be some
1.2944 + // call to enable/disable allocation profiling in a generation/space,
1.2945 + // and the iterator ought to return the objects allocated in the
1.2946 + // gen/space since the enable call, or the last iterator call (which
1.2947 + // will probably be at a GC.) That way, for gens like CM&S that would
1.2948 + // require some extra data structure to support this, we only pay the
1.2949 + // cost when it's in use...
1.2950 + cmsSpace()->object_iterate_since_last_GC(blk);
1.2951 +}
1.2952 +
1.2953 +void
1.2954 +ConcurrentMarkSweepGeneration::younger_refs_iterate(OopsInGenClosure* cl) {
1.2955 + cl->set_generation(this);
1.2956 + younger_refs_in_space_iterate(_cmsSpace, cl);
1.2957 + cl->reset_generation();
1.2958 +}
1.2959 +
1.2960 +void
1.2961 +ConcurrentMarkSweepGeneration::oop_iterate(MemRegion mr, OopClosure* cl) {
1.2962 + if (freelistLock()->owned_by_self()) {
1.2963 + Generation::oop_iterate(mr, cl);
1.2964 + } else {
1.2965 + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
1.2966 + Generation::oop_iterate(mr, cl);
1.2967 + }
1.2968 +}
1.2969 +
1.2970 +void
1.2971 +ConcurrentMarkSweepGeneration::oop_iterate(OopClosure* cl) {
1.2972 + if (freelistLock()->owned_by_self()) {
1.2973 + Generation::oop_iterate(cl);
1.2974 + } else {
1.2975 + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
1.2976 + Generation::oop_iterate(cl);
1.2977 + }
1.2978 +}
1.2979 +
1.2980 +void
1.2981 +ConcurrentMarkSweepGeneration::object_iterate(ObjectClosure* cl) {
1.2982 + if (freelistLock()->owned_by_self()) {
1.2983 + Generation::object_iterate(cl);
1.2984 + } else {
1.2985 + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
1.2986 + Generation::object_iterate(cl);
1.2987 + }
1.2988 +}
1.2989 +
1.2990 +void
1.2991 +ConcurrentMarkSweepGeneration::pre_adjust_pointers() {
1.2992 +}
1.2993 +
1.2994 +void
1.2995 +ConcurrentMarkSweepGeneration::post_compact() {
1.2996 +}
1.2997 +
1.2998 +void
1.2999 +ConcurrentMarkSweepGeneration::prepare_for_verify() {
1.3000 + // Fix the linear allocation blocks to look like free blocks.
1.3001 +
1.3002 + // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those
1.3003 + // are not called when the heap is verified during universe initialization and
1.3004 + // at vm shutdown.
1.3005 + if (freelistLock()->owned_by_self()) {
1.3006 + cmsSpace()->prepare_for_verify();
1.3007 + } else {
1.3008 + MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag);
1.3009 + cmsSpace()->prepare_for_verify();
1.3010 + }
1.3011 +}
1.3012 +
1.3013 +void
1.3014 +ConcurrentMarkSweepGeneration::verify(bool allow_dirty /* ignored */) {
1.3015 + // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those
1.3016 + // are not called when the heap is verified during universe initialization and
1.3017 + // at vm shutdown.
1.3018 + if (freelistLock()->owned_by_self()) {
1.3019 + cmsSpace()->verify(false /* ignored */);
1.3020 + } else {
1.3021 + MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag);
1.3022 + cmsSpace()->verify(false /* ignored */);
1.3023 + }
1.3024 +}
1.3025 +
1.3026 +void CMSCollector::verify(bool allow_dirty /* ignored */) {
1.3027 + _cmsGen->verify(allow_dirty);
1.3028 + _permGen->verify(allow_dirty);
1.3029 +}
1.3030 +
1.3031 +#ifndef PRODUCT
1.3032 +bool CMSCollector::overflow_list_is_empty() const {
1.3033 + assert(_num_par_pushes >= 0, "Inconsistency");
1.3034 + if (_overflow_list == NULL) {
1.3035 + assert(_num_par_pushes == 0, "Inconsistency");
1.3036 + }
1.3037 + return _overflow_list == NULL;
1.3038 +}
1.3039 +
1.3040 +// The methods verify_work_stacks_empty() and verify_overflow_empty()
1.3041 +// merely consolidate assertion checks that appear to occur together frequently.
1.3042 +void CMSCollector::verify_work_stacks_empty() const {
1.3043 + assert(_markStack.isEmpty(), "Marking stack should be empty");
1.3044 + assert(overflow_list_is_empty(), "Overflow list should be empty");
1.3045 +}
1.3046 +
1.3047 +void CMSCollector::verify_overflow_empty() const {
1.3048 + assert(overflow_list_is_empty(), "Overflow list should be empty");
1.3049 + assert(no_preserved_marks(), "No preserved marks");
1.3050 +}
1.3051 +#endif // PRODUCT
1.3052 +
1.3053 +void CMSCollector::setup_cms_unloading_and_verification_state() {
1.3054 + const bool should_verify = VerifyBeforeGC || VerifyAfterGC || VerifyDuringGC
1.3055 + || VerifyBeforeExit;
1.3056 + const int rso = SharedHeap::SO_Symbols | SharedHeap::SO_Strings
1.3057 + | SharedHeap::SO_CodeCache;
1.3058 +
1.3059 + if (cms_should_unload_classes()) { // Should unload classes this cycle
1.3060 + remove_root_scanning_option(rso); // Shrink the root set appropriately
1.3061 + set_verifying(should_verify); // Set verification state for this cycle
1.3062 + return; // Nothing else needs to be done at this time
1.3063 + }
1.3064 +
1.3065 + // Not unloading classes this cycle
1.3066 + assert(!cms_should_unload_classes(), "Inconsitency!");
1.3067 + if ((!verifying() || cms_unloaded_classes_last_cycle()) && should_verify) {
1.3068 + // We were not verifying, or we _were_ unloading classes in the last cycle,
1.3069 + // AND some verification options are enabled this cycle; in this case,
1.3070 + // we must make sure that the deadness map is allocated if not already so,
1.3071 + // and cleared (if already allocated previously --
1.3072 + // CMSBitMap::sizeInBits() is used to determine if it's allocated).
1.3073 + if (perm_gen_verify_bit_map()->sizeInBits() == 0) {
1.3074 + if (!perm_gen_verify_bit_map()->allocate(_permGen->reserved())) {
1.3075 + warning("Failed to allocate permanent generation verification CMS Bit Map;\n"
1.3076 + "permanent generation verification disabled");
1.3077 + return; // Note that we leave verification disabled, so we'll retry this
1.3078 + // allocation next cycle. We _could_ remember this failure
1.3079 + // and skip further attempts and permanently disable verification
1.3080 + // attempts if that is considered more desirable.
1.3081 + }
1.3082 + assert(perm_gen_verify_bit_map()->covers(_permGen->reserved()),
1.3083 + "_perm_gen_ver_bit_map inconsistency?");
1.3084 + } else {
1.3085 + perm_gen_verify_bit_map()->clear_all();
1.3086 + }
1.3087 + // Include symbols, strings and code cache elements to prevent their resurrection.
1.3088 + add_root_scanning_option(rso);
1.3089 + set_verifying(true);
1.3090 + } else if (verifying() && !should_verify) {
1.3091 + // We were verifying, but some verification flags got disabled.
1.3092 + set_verifying(false);
1.3093 + // Exclude symbols, strings and code cache elements from root scanning to
1.3094 + // reduce IM and RM pauses.
1.3095 + remove_root_scanning_option(rso);
1.3096 + }
1.3097 +}
1.3098 +
1.3099 +
1.3100 +#ifndef PRODUCT
1.3101 +HeapWord* CMSCollector::block_start(const void* p) const {
1.3102 + const HeapWord* addr = (HeapWord*)p;
1.3103 + if (_span.contains(p)) {
1.3104 + if (_cmsGen->cmsSpace()->is_in_reserved(addr)) {
1.3105 + return _cmsGen->cmsSpace()->block_start(p);
1.3106 + } else {
1.3107 + assert(_permGen->cmsSpace()->is_in_reserved(addr),
1.3108 + "Inconsistent _span?");
1.3109 + return _permGen->cmsSpace()->block_start(p);
1.3110 + }
1.3111 + }
1.3112 + return NULL;
1.3113 +}
1.3114 +#endif
1.3115 +
1.3116 +HeapWord*
1.3117 +ConcurrentMarkSweepGeneration::expand_and_allocate(size_t word_size,
1.3118 + bool tlab,
1.3119 + bool parallel) {
1.3120 + assert(!tlab, "Can't deal with TLAB allocation");
1.3121 + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
1.3122 + expand(word_size*HeapWordSize, MinHeapDeltaBytes,
1.3123 + CMSExpansionCause::_satisfy_allocation);
1.3124 + if (GCExpandToAllocateDelayMillis > 0) {
1.3125 + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
1.3126 + }
1.3127 + size_t adj_word_sz = CompactibleFreeListSpace::adjustObjectSize(word_size);
1.3128 + if (parallel) {
1.3129 + return cmsSpace()->par_allocate(adj_word_sz);
1.3130 + } else {
1.3131 + return cmsSpace()->allocate(adj_word_sz);
1.3132 + }
1.3133 +}
1.3134 +
1.3135 +// YSR: All of this generation expansion/shrinking stuff is an exact copy of
1.3136 +// OneContigSpaceCardGeneration, which makes me wonder if we should move this
1.3137 +// to CardGeneration and share it...
1.3138 +void ConcurrentMarkSweepGeneration::expand(size_t bytes, size_t expand_bytes,
1.3139 + CMSExpansionCause::Cause cause)
1.3140 +{
1.3141 + assert_locked_or_safepoint(Heap_lock);
1.3142 +
1.3143 + size_t aligned_bytes = ReservedSpace::page_align_size_up(bytes);
1.3144 + size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes);
1.3145 + bool success = false;
1.3146 + if (aligned_expand_bytes > aligned_bytes) {
1.3147 + success = grow_by(aligned_expand_bytes);
1.3148 + }
1.3149 + if (!success) {
1.3150 + success = grow_by(aligned_bytes);
1.3151 + }
1.3152 + if (!success) {
1.3153 + size_t remaining_bytes = _virtual_space.uncommitted_size();
1.3154 + if (remaining_bytes > 0) {
1.3155 + success = grow_by(remaining_bytes);
1.3156 + }
1.3157 + }
1.3158 + if (GC_locker::is_active()) {
1.3159 + if (PrintGC && Verbose) {
1.3160 + gclog_or_tty->print_cr("Garbage collection disabled, expanded heap instead");
1.3161 + }
1.3162 + }
1.3163 + // remember why we expanded; this information is used
1.3164 + // by shouldConcurrentCollect() when making decisions on whether to start
1.3165 + // a new CMS cycle.
1.3166 + if (success) {
1.3167 + set_expansion_cause(cause);
1.3168 + if (PrintGCDetails && Verbose) {
1.3169 + gclog_or_tty->print_cr("Expanded CMS gen for %s",
1.3170 + CMSExpansionCause::to_string(cause));
1.3171 + }
1.3172 + }
1.3173 +}
1.3174 +
1.3175 +HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) {
1.3176 + HeapWord* res = NULL;
1.3177 + MutexLocker x(ParGCRareEvent_lock);
1.3178 + while (true) {
1.3179 + // Expansion by some other thread might make alloc OK now:
1.3180 + res = ps->lab.alloc(word_sz);
1.3181 + if (res != NULL) return res;
1.3182 + // If there's not enough expansion space available, give up.
1.3183 + if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) {
1.3184 + return NULL;
1.3185 + }
1.3186 + // Otherwise, we try expansion.
1.3187 + expand(word_sz*HeapWordSize, MinHeapDeltaBytes,
1.3188 + CMSExpansionCause::_allocate_par_lab);
1.3189 + // Now go around the loop and try alloc again;
1.3190 + // A competing par_promote might beat us to the expansion space,
1.3191 + // so we may go around the loop again if promotion fails agaion.
1.3192 + if (GCExpandToAllocateDelayMillis > 0) {
1.3193 + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
1.3194 + }
1.3195 + }
1.3196 +}
1.3197 +
1.3198 +
1.3199 +bool ConcurrentMarkSweepGeneration::expand_and_ensure_spooling_space(
1.3200 + PromotionInfo* promo) {
1.3201 + MutexLocker x(ParGCRareEvent_lock);
1.3202 + size_t refill_size_bytes = promo->refillSize() * HeapWordSize;
1.3203 + while (true) {
1.3204 + // Expansion by some other thread might make alloc OK now:
1.3205 + if (promo->ensure_spooling_space()) {
1.3206 + assert(promo->has_spooling_space(),
1.3207 + "Post-condition of successful ensure_spooling_space()");
1.3208 + return true;
1.3209 + }
1.3210 + // If there's not enough expansion space available, give up.
1.3211 + if (_virtual_space.uncommitted_size() < refill_size_bytes) {
1.3212 + return false;
1.3213 + }
1.3214 + // Otherwise, we try expansion.
1.3215 + expand(refill_size_bytes, MinHeapDeltaBytes,
1.3216 + CMSExpansionCause::_allocate_par_spooling_space);
1.3217 + // Now go around the loop and try alloc again;
1.3218 + // A competing allocation might beat us to the expansion space,
1.3219 + // so we may go around the loop again if allocation fails again.
1.3220 + if (GCExpandToAllocateDelayMillis > 0) {
1.3221 + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
1.3222 + }
1.3223 + }
1.3224 +}
1.3225 +
1.3226 +
1.3227 +
1.3228 +void ConcurrentMarkSweepGeneration::shrink(size_t bytes) {
1.3229 + assert_locked_or_safepoint(Heap_lock);
1.3230 + size_t size = ReservedSpace::page_align_size_down(bytes);
1.3231 + if (size > 0) {
1.3232 + shrink_by(size);
1.3233 + }
1.3234 +}
1.3235 +
1.3236 +bool ConcurrentMarkSweepGeneration::grow_by(size_t bytes) {
1.3237 + assert_locked_or_safepoint(Heap_lock);
1.3238 + bool result = _virtual_space.expand_by(bytes);
1.3239 + if (result) {
1.3240 + HeapWord* old_end = _cmsSpace->end();
1.3241 + size_t new_word_size =
1.3242 + heap_word_size(_virtual_space.committed_size());
1.3243 + MemRegion mr(_cmsSpace->bottom(), new_word_size);
1.3244 + _bts->resize(new_word_size); // resize the block offset shared array
1.3245 + Universe::heap()->barrier_set()->resize_covered_region(mr);
1.3246 + // Hmmmm... why doesn't CFLS::set_end verify locking?
1.3247 + // This is quite ugly; FIX ME XXX
1.3248 + _cmsSpace->assert_locked();
1.3249 + _cmsSpace->set_end((HeapWord*)_virtual_space.high());
1.3250 +
1.3251 + // update the space and generation capacity counters
1.3252 + if (UsePerfData) {
1.3253 + _space_counters->update_capacity();
1.3254 + _gen_counters->update_all();
1.3255 + }
1.3256 +
1.3257 + if (Verbose && PrintGC) {
1.3258 + size_t new_mem_size = _virtual_space.committed_size();
1.3259 + size_t old_mem_size = new_mem_size - bytes;
1.3260 + gclog_or_tty->print_cr("Expanding %s from %ldK by %ldK to %ldK",
1.3261 + name(), old_mem_size/K, bytes/K, new_mem_size/K);
1.3262 + }
1.3263 + }
1.3264 + return result;
1.3265 +}
1.3266 +
1.3267 +bool ConcurrentMarkSweepGeneration::grow_to_reserved() {
1.3268 + assert_locked_or_safepoint(Heap_lock);
1.3269 + bool success = true;
1.3270 + const size_t remaining_bytes = _virtual_space.uncommitted_size();
1.3271 + if (remaining_bytes > 0) {
1.3272 + success = grow_by(remaining_bytes);
1.3273 + DEBUG_ONLY(if (!success) warning("grow to reserved failed");)
1.3274 + }
1.3275 + return success;
1.3276 +}
1.3277 +
1.3278 +void ConcurrentMarkSweepGeneration::shrink_by(size_t bytes) {
1.3279 + assert_locked_or_safepoint(Heap_lock);
1.3280 + assert_lock_strong(freelistLock());
1.3281 + // XXX Fix when compaction is implemented.
1.3282 + warning("Shrinking of CMS not yet implemented");
1.3283 + return;
1.3284 +}
1.3285 +
1.3286 +
1.3287 +// Simple ctor/dtor wrapper for accounting & timer chores around concurrent
1.3288 +// phases.
1.3289 +class CMSPhaseAccounting: public StackObj {
1.3290 + public:
1.3291 + CMSPhaseAccounting(CMSCollector *collector,
1.3292 + const char *phase,
1.3293 + bool print_cr = true);
1.3294 + ~CMSPhaseAccounting();
1.3295 +
1.3296 + private:
1.3297 + CMSCollector *_collector;
1.3298 + const char *_phase;
1.3299 + elapsedTimer _wallclock;
1.3300 + bool _print_cr;
1.3301 +
1.3302 + public:
1.3303 + // Not MT-safe; so do not pass around these StackObj's
1.3304 + // where they may be accessed by other threads.
1.3305 + jlong wallclock_millis() {
1.3306 + assert(_wallclock.is_active(), "Wall clock should not stop");
1.3307 + _wallclock.stop(); // to record time
1.3308 + jlong ret = _wallclock.milliseconds();
1.3309 + _wallclock.start(); // restart
1.3310 + return ret;
1.3311 + }
1.3312 +};
1.3313 +
1.3314 +CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector,
1.3315 + const char *phase,
1.3316 + bool print_cr) :
1.3317 + _collector(collector), _phase(phase), _print_cr(print_cr) {
1.3318 +
1.3319 + if (PrintCMSStatistics != 0) {
1.3320 + _collector->resetYields();
1.3321 + }
1.3322 + if (PrintGCDetails && PrintGCTimeStamps) {
1.3323 + gclog_or_tty->date_stamp(PrintGCDateStamps);
1.3324 + gclog_or_tty->stamp();
1.3325 + gclog_or_tty->print_cr(": [%s-concurrent-%s-start]",
1.3326 + _collector->cmsGen()->short_name(), _phase);
1.3327 + }
1.3328 + _collector->resetTimer();
1.3329 + _wallclock.start();
1.3330 + _collector->startTimer();
1.3331 +}
1.3332 +
1.3333 +CMSPhaseAccounting::~CMSPhaseAccounting() {
1.3334 + assert(_wallclock.is_active(), "Wall clock should not have stopped");
1.3335 + _collector->stopTimer();
1.3336 + _wallclock.stop();
1.3337 + if (PrintGCDetails) {
1.3338 + gclog_or_tty->date_stamp(PrintGCDateStamps);
1.3339 + if (PrintGCTimeStamps) {
1.3340 + gclog_or_tty->stamp();
1.3341 + gclog_or_tty->print(": ");
1.3342 + }
1.3343 + gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]",
1.3344 + _collector->cmsGen()->short_name(),
1.3345 + _phase, _collector->timerValue(), _wallclock.seconds());
1.3346 + if (_print_cr) {
1.3347 + gclog_or_tty->print_cr("");
1.3348 + }
1.3349 + if (PrintCMSStatistics != 0) {
1.3350 + gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase,
1.3351 + _collector->yields());
1.3352 + }
1.3353 + }
1.3354 +}
1.3355 +
1.3356 +// CMS work
1.3357 +
1.3358 +// Checkpoint the roots into this generation from outside
1.3359 +// this generation. [Note this initial checkpoint need only
1.3360 +// be approximate -- we'll do a catch up phase subsequently.]
1.3361 +void CMSCollector::checkpointRootsInitial(bool asynch) {
1.3362 + assert(_collectorState == InitialMarking, "Wrong collector state");
1.3363 + check_correct_thread_executing();
1.3364 + ReferenceProcessor* rp = ref_processor();
1.3365 + SpecializationStats::clear();
1.3366 + assert(_restart_addr == NULL, "Control point invariant");
1.3367 + if (asynch) {
1.3368 + // acquire locks for subsequent manipulations
1.3369 + MutexLockerEx x(bitMapLock(),
1.3370 + Mutex::_no_safepoint_check_flag);
1.3371 + checkpointRootsInitialWork(asynch);
1.3372 + rp->verify_no_references_recorded();
1.3373 + rp->enable_discovery(); // enable ("weak") refs discovery
1.3374 + _collectorState = Marking;
1.3375 + } else {
1.3376 + // (Weak) Refs discovery: this is controlled from genCollectedHeap::do_collection
1.3377 + // which recognizes if we are a CMS generation, and doesn't try to turn on
1.3378 + // discovery; verify that they aren't meddling.
1.3379 + assert(!rp->discovery_is_atomic(),
1.3380 + "incorrect setting of discovery predicate");
1.3381 + assert(!rp->discovery_enabled(), "genCollectedHeap shouldn't control "
1.3382 + "ref discovery for this generation kind");
1.3383 + // already have locks
1.3384 + checkpointRootsInitialWork(asynch);
1.3385 + rp->enable_discovery(); // now enable ("weak") refs discovery
1.3386 + _collectorState = Marking;
1.3387 + }
1.3388 + SpecializationStats::print();
1.3389 +}
1.3390 +
1.3391 +void CMSCollector::checkpointRootsInitialWork(bool asynch) {
1.3392 + assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
1.3393 + assert(_collectorState == InitialMarking, "just checking");
1.3394 +
1.3395 + // If there has not been a GC[n-1] since last GC[n] cycle completed,
1.3396 + // precede our marking with a collection of all
1.3397 + // younger generations to keep floating garbage to a minimum.
1.3398 + // XXX: we won't do this for now -- it's an optimization to be done later.
1.3399 +
1.3400 + // already have locks
1.3401 + assert_lock_strong(bitMapLock());
1.3402 + assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
1.3403 +
1.3404 + // Setup the verification and class unloading state for this
1.3405 + // CMS collection cycle.
1.3406 + setup_cms_unloading_and_verification_state();
1.3407 +
1.3408 + NOT_PRODUCT(TraceTime t("\ncheckpointRootsInitialWork",
1.3409 + PrintGCDetails && Verbose, true, gclog_or_tty);)
1.3410 + if (UseAdaptiveSizePolicy) {
1.3411 + size_policy()->checkpoint_roots_initial_begin();
1.3412 + }
1.3413 +
1.3414 + // Reset all the PLAB chunk arrays if necessary.
1.3415 + if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
1.3416 + reset_survivor_plab_arrays();
1.3417 + }
1.3418 +
1.3419 + ResourceMark rm;
1.3420 + HandleMark hm;
1.3421 +
1.3422 + FalseClosure falseClosure;
1.3423 + // In the case of a synchronous collection, we will elide the
1.3424 + // remark step, so it's important to catch all the nmethod oops
1.3425 + // in this step; hence the last argument to the constrcutor below.
1.3426 + MarkRefsIntoClosure notOlder(_span, &_markBitMap, !asynch /* nmethods */);
1.3427 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.3428 +
1.3429 + verify_work_stacks_empty();
1.3430 + verify_overflow_empty();
1.3431 +
1.3432 + gch->ensure_parsability(false); // fill TLABs, but no need to retire them
1.3433 + // Update the saved marks which may affect the root scans.
1.3434 + gch->save_marks();
1.3435 +
1.3436 + // weak reference processing has not started yet.
1.3437 + ref_processor()->set_enqueuing_is_done(false);
1.3438 +
1.3439 + {
1.3440 + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
1.3441 + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
1.3442 + gch->gen_process_strong_roots(_cmsGen->level(),
1.3443 + true, // younger gens are roots
1.3444 + true, // collecting perm gen
1.3445 + SharedHeap::ScanningOption(roots_scanning_options()),
1.3446 + NULL, ¬Older);
1.3447 + }
1.3448 +
1.3449 + // Clear mod-union table; it will be dirtied in the prologue of
1.3450 + // CMS generation per each younger generation collection.
1.3451 +
1.3452 + assert(_modUnionTable.isAllClear(),
1.3453 + "Was cleared in most recent final checkpoint phase"
1.3454 + " or no bits are set in the gc_prologue before the start of the next "
1.3455 + "subsequent marking phase.");
1.3456 +
1.3457 + // Temporarily disabled, since pre/post-consumption closures don't
1.3458 + // care about precleaned cards
1.3459 + #if 0
1.3460 + {
1.3461 + MemRegion mr = MemRegion((HeapWord*)_virtual_space.low(),
1.3462 + (HeapWord*)_virtual_space.high());
1.3463 + _ct->ct_bs()->preclean_dirty_cards(mr);
1.3464 + }
1.3465 + #endif
1.3466 +
1.3467 + // Save the end of the used_region of the constituent generations
1.3468 + // to be used to limit the extent of sweep in each generation.
1.3469 + save_sweep_limits();
1.3470 + if (UseAdaptiveSizePolicy) {
1.3471 + size_policy()->checkpoint_roots_initial_end(gch->gc_cause());
1.3472 + }
1.3473 + verify_overflow_empty();
1.3474 +}
1.3475 +
1.3476 +bool CMSCollector::markFromRoots(bool asynch) {
1.3477 + // we might be tempted to assert that:
1.3478 + // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1.3479 + // "inconsistent argument?");
1.3480 + // However that wouldn't be right, because it's possible that
1.3481 + // a safepoint is indeed in progress as a younger generation
1.3482 + // stop-the-world GC happens even as we mark in this generation.
1.3483 + assert(_collectorState == Marking, "inconsistent state?");
1.3484 + check_correct_thread_executing();
1.3485 + verify_overflow_empty();
1.3486 +
1.3487 + bool res;
1.3488 + if (asynch) {
1.3489 +
1.3490 + // Start the timers for adaptive size policy for the concurrent phases
1.3491 + // Do it here so that the foreground MS can use the concurrent
1.3492 + // timer since a foreground MS might has the sweep done concurrently
1.3493 + // or STW.
1.3494 + if (UseAdaptiveSizePolicy) {
1.3495 + size_policy()->concurrent_marking_begin();
1.3496 + }
1.3497 +
1.3498 + // Weak ref discovery note: We may be discovering weak
1.3499 + // refs in this generation concurrent (but interleaved) with
1.3500 + // weak ref discovery by a younger generation collector.
1.3501 +
1.3502 + CMSTokenSyncWithLocks ts(true, bitMapLock());
1.3503 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.3504 + CMSPhaseAccounting pa(this, "mark", !PrintGCDetails);
1.3505 + res = markFromRootsWork(asynch);
1.3506 + if (res) {
1.3507 + _collectorState = Precleaning;
1.3508 + } else { // We failed and a foreground collection wants to take over
1.3509 + assert(_foregroundGCIsActive, "internal state inconsistency");
1.3510 + assert(_restart_addr == NULL, "foreground will restart from scratch");
1.3511 + if (PrintGCDetails) {
1.3512 + gclog_or_tty->print_cr("bailing out to foreground collection");
1.3513 + }
1.3514 + }
1.3515 + if (UseAdaptiveSizePolicy) {
1.3516 + size_policy()->concurrent_marking_end();
1.3517 + }
1.3518 + } else {
1.3519 + assert(SafepointSynchronize::is_at_safepoint(),
1.3520 + "inconsistent with asynch == false");
1.3521 + if (UseAdaptiveSizePolicy) {
1.3522 + size_policy()->ms_collection_marking_begin();
1.3523 + }
1.3524 + // already have locks
1.3525 + res = markFromRootsWork(asynch);
1.3526 + _collectorState = FinalMarking;
1.3527 + if (UseAdaptiveSizePolicy) {
1.3528 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.3529 + size_policy()->ms_collection_marking_end(gch->gc_cause());
1.3530 + }
1.3531 + }
1.3532 + verify_overflow_empty();
1.3533 + return res;
1.3534 +}
1.3535 +
1.3536 +bool CMSCollector::markFromRootsWork(bool asynch) {
1.3537 + // iterate over marked bits in bit map, doing a full scan and mark
1.3538 + // from these roots using the following algorithm:
1.3539 + // . if oop is to the right of the current scan pointer,
1.3540 + // mark corresponding bit (we'll process it later)
1.3541 + // . else (oop is to left of current scan pointer)
1.3542 + // push oop on marking stack
1.3543 + // . drain the marking stack
1.3544 +
1.3545 + // Note that when we do a marking step we need to hold the
1.3546 + // bit map lock -- recall that direct allocation (by mutators)
1.3547 + // and promotion (by younger generation collectors) is also
1.3548 + // marking the bit map. [the so-called allocate live policy.]
1.3549 + // Because the implementation of bit map marking is not
1.3550 + // robust wrt simultaneous marking of bits in the same word,
1.3551 + // we need to make sure that there is no such interference
1.3552 + // between concurrent such updates.
1.3553 +
1.3554 + // already have locks
1.3555 + assert_lock_strong(bitMapLock());
1.3556 +
1.3557 + // Clear the revisit stack, just in case there are any
1.3558 + // obsolete contents from a short-circuited previous CMS cycle.
1.3559 + _revisitStack.reset();
1.3560 + verify_work_stacks_empty();
1.3561 + verify_overflow_empty();
1.3562 + assert(_revisitStack.isEmpty(), "tabula rasa");
1.3563 +
1.3564 + bool result = false;
1.3565 + if (CMSConcurrentMTEnabled && ParallelCMSThreads > 0) {
1.3566 + result = do_marking_mt(asynch);
1.3567 + } else {
1.3568 + result = do_marking_st(asynch);
1.3569 + }
1.3570 + return result;
1.3571 +}
1.3572 +
1.3573 +// Forward decl
1.3574 +class CMSConcMarkingTask;
1.3575 +
1.3576 +class CMSConcMarkingTerminator: public ParallelTaskTerminator {
1.3577 + CMSCollector* _collector;
1.3578 + CMSConcMarkingTask* _task;
1.3579 + bool _yield;
1.3580 + protected:
1.3581 + virtual void yield();
1.3582 + public:
1.3583 + // "n_threads" is the number of threads to be terminated.
1.3584 + // "queue_set" is a set of work queues of other threads.
1.3585 + // "collector" is the CMS collector associated with this task terminator.
1.3586 + // "yield" indicates whether we need the gang as a whole to yield.
1.3587 + CMSConcMarkingTerminator(int n_threads, TaskQueueSetSuper* queue_set,
1.3588 + CMSCollector* collector, bool yield) :
1.3589 + ParallelTaskTerminator(n_threads, queue_set),
1.3590 + _collector(collector),
1.3591 + _yield(yield) { }
1.3592 +
1.3593 + void set_task(CMSConcMarkingTask* task) {
1.3594 + _task = task;
1.3595 + }
1.3596 +};
1.3597 +
1.3598 +// MT Concurrent Marking Task
1.3599 +class CMSConcMarkingTask: public YieldingFlexibleGangTask {
1.3600 + CMSCollector* _collector;
1.3601 + YieldingFlexibleWorkGang* _workers; // the whole gang
1.3602 + int _n_workers; // requested/desired # workers
1.3603 + bool _asynch;
1.3604 + bool _result;
1.3605 + CompactibleFreeListSpace* _cms_space;
1.3606 + CompactibleFreeListSpace* _perm_space;
1.3607 + HeapWord* _global_finger;
1.3608 +
1.3609 + // Exposed here for yielding support
1.3610 + Mutex* const _bit_map_lock;
1.3611 +
1.3612 + // The per thread work queues, available here for stealing
1.3613 + OopTaskQueueSet* _task_queues;
1.3614 + CMSConcMarkingTerminator _term;
1.3615 +
1.3616 + public:
1.3617 + CMSConcMarkingTask(CMSCollector* collector,
1.3618 + CompactibleFreeListSpace* cms_space,
1.3619 + CompactibleFreeListSpace* perm_space,
1.3620 + bool asynch, int n_workers,
1.3621 + YieldingFlexibleWorkGang* workers,
1.3622 + OopTaskQueueSet* task_queues):
1.3623 + YieldingFlexibleGangTask("Concurrent marking done multi-threaded"),
1.3624 + _collector(collector),
1.3625 + _cms_space(cms_space),
1.3626 + _perm_space(perm_space),
1.3627 + _asynch(asynch), _n_workers(n_workers), _result(true),
1.3628 + _workers(workers), _task_queues(task_queues),
1.3629 + _term(n_workers, task_queues, _collector, asynch),
1.3630 + _bit_map_lock(collector->bitMapLock())
1.3631 + {
1.3632 + assert(n_workers <= workers->total_workers(),
1.3633 + "Else termination won't work correctly today"); // XXX FIX ME!
1.3634 + _requested_size = n_workers;
1.3635 + _term.set_task(this);
1.3636 + assert(_cms_space->bottom() < _perm_space->bottom(),
1.3637 + "Finger incorrectly initialized below");
1.3638 + _global_finger = _cms_space->bottom();
1.3639 + }
1.3640 +
1.3641 +
1.3642 + OopTaskQueueSet* task_queues() { return _task_queues; }
1.3643 +
1.3644 + OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
1.3645 +
1.3646 + HeapWord** global_finger_addr() { return &_global_finger; }
1.3647 +
1.3648 + CMSConcMarkingTerminator* terminator() { return &_term; }
1.3649 +
1.3650 + void work(int i);
1.3651 +
1.3652 + virtual void coordinator_yield(); // stuff done by coordinator
1.3653 + bool result() { return _result; }
1.3654 +
1.3655 + void reset(HeapWord* ra) {
1.3656 + _term.reset_for_reuse();
1.3657 + }
1.3658 +
1.3659 + static bool get_work_from_overflow_stack(CMSMarkStack* ovflw_stk,
1.3660 + OopTaskQueue* work_q);
1.3661 +
1.3662 + private:
1.3663 + void do_scan_and_mark(int i, CompactibleFreeListSpace* sp);
1.3664 + void do_work_steal(int i);
1.3665 + void bump_global_finger(HeapWord* f);
1.3666 +};
1.3667 +
1.3668 +void CMSConcMarkingTerminator::yield() {
1.3669 + if (ConcurrentMarkSweepThread::should_yield() &&
1.3670 + !_collector->foregroundGCIsActive() &&
1.3671 + _yield) {
1.3672 + _task->yield();
1.3673 + } else {
1.3674 + ParallelTaskTerminator::yield();
1.3675 + }
1.3676 +}
1.3677 +
1.3678 +////////////////////////////////////////////////////////////////
1.3679 +// Concurrent Marking Algorithm Sketch
1.3680 +////////////////////////////////////////////////////////////////
1.3681 +// Until all tasks exhausted (both spaces):
1.3682 +// -- claim next available chunk
1.3683 +// -- bump global finger via CAS
1.3684 +// -- find first object that starts in this chunk
1.3685 +// and start scanning bitmap from that position
1.3686 +// -- scan marked objects for oops
1.3687 +// -- CAS-mark target, and if successful:
1.3688 +// . if target oop is above global finger (volatile read)
1.3689 +// nothing to do
1.3690 +// . if target oop is in chunk and above local finger
1.3691 +// then nothing to do
1.3692 +// . else push on work-queue
1.3693 +// -- Deal with possible overflow issues:
1.3694 +// . local work-queue overflow causes stuff to be pushed on
1.3695 +// global (common) overflow queue
1.3696 +// . always first empty local work queue
1.3697 +// . then get a batch of oops from global work queue if any
1.3698 +// . then do work stealing
1.3699 +// -- When all tasks claimed (both spaces)
1.3700 +// and local work queue empty,
1.3701 +// then in a loop do:
1.3702 +// . check global overflow stack; steal a batch of oops and trace
1.3703 +// . try to steal from other threads oif GOS is empty
1.3704 +// . if neither is available, offer termination
1.3705 +// -- Terminate and return result
1.3706 +//
1.3707 +void CMSConcMarkingTask::work(int i) {
1.3708 + elapsedTimer _timer;
1.3709 + ResourceMark rm;
1.3710 + HandleMark hm;
1.3711 +
1.3712 + DEBUG_ONLY(_collector->verify_overflow_empty();)
1.3713 +
1.3714 + // Before we begin work, our work queue should be empty
1.3715 + assert(work_queue(i)->size() == 0, "Expected to be empty");
1.3716 + // Scan the bitmap covering _cms_space, tracing through grey objects.
1.3717 + _timer.start();
1.3718 + do_scan_and_mark(i, _cms_space);
1.3719 + _timer.stop();
1.3720 + if (PrintCMSStatistics != 0) {
1.3721 + gclog_or_tty->print_cr("Finished cms space scanning in %dth thread: %3.3f sec",
1.3722 + i, _timer.seconds()); // XXX: need xxx/xxx type of notation, two timers
1.3723 + }
1.3724 +
1.3725 + // ... do the same for the _perm_space
1.3726 + _timer.reset();
1.3727 + _timer.start();
1.3728 + do_scan_and_mark(i, _perm_space);
1.3729 + _timer.stop();
1.3730 + if (PrintCMSStatistics != 0) {
1.3731 + gclog_or_tty->print_cr("Finished perm space scanning in %dth thread: %3.3f sec",
1.3732 + i, _timer.seconds()); // XXX: need xxx/xxx type of notation, two timers
1.3733 + }
1.3734 +
1.3735 + // ... do work stealing
1.3736 + _timer.reset();
1.3737 + _timer.start();
1.3738 + do_work_steal(i);
1.3739 + _timer.stop();
1.3740 + if (PrintCMSStatistics != 0) {
1.3741 + gclog_or_tty->print_cr("Finished work stealing in %dth thread: %3.3f sec",
1.3742 + i, _timer.seconds()); // XXX: need xxx/xxx type of notation, two timers
1.3743 + }
1.3744 + assert(_collector->_markStack.isEmpty(), "Should have been emptied");
1.3745 + assert(work_queue(i)->size() == 0, "Should have been emptied");
1.3746 + // Note that under the current task protocol, the
1.3747 + // following assertion is true even of the spaces
1.3748 + // expanded since the completion of the concurrent
1.3749 + // marking. XXX This will likely change under a strict
1.3750 + // ABORT semantics.
1.3751 + assert(_global_finger > _cms_space->end() &&
1.3752 + _global_finger >= _perm_space->end(),
1.3753 + "All tasks have been completed");
1.3754 + DEBUG_ONLY(_collector->verify_overflow_empty();)
1.3755 +}
1.3756 +
1.3757 +void CMSConcMarkingTask::bump_global_finger(HeapWord* f) {
1.3758 + HeapWord* read = _global_finger;
1.3759 + HeapWord* cur = read;
1.3760 + while (f > read) {
1.3761 + cur = read;
1.3762 + read = (HeapWord*) Atomic::cmpxchg_ptr(f, &_global_finger, cur);
1.3763 + if (cur == read) {
1.3764 + // our cas succeeded
1.3765 + assert(_global_finger >= f, "protocol consistency");
1.3766 + break;
1.3767 + }
1.3768 + }
1.3769 +}
1.3770 +
1.3771 +// This is really inefficient, and should be redone by
1.3772 +// using (not yet available) block-read and -write interfaces to the
1.3773 +// stack and the work_queue. XXX FIX ME !!!
1.3774 +bool CMSConcMarkingTask::get_work_from_overflow_stack(CMSMarkStack* ovflw_stk,
1.3775 + OopTaskQueue* work_q) {
1.3776 + // Fast lock-free check
1.3777 + if (ovflw_stk->length() == 0) {
1.3778 + return false;
1.3779 + }
1.3780 + assert(work_q->size() == 0, "Shouldn't steal");
1.3781 + MutexLockerEx ml(ovflw_stk->par_lock(),
1.3782 + Mutex::_no_safepoint_check_flag);
1.3783 + // Grab up to 1/4 the size of the work queue
1.3784 + size_t num = MIN2((size_t)work_q->max_elems()/4,
1.3785 + (size_t)ParGCDesiredObjsFromOverflowList);
1.3786 + num = MIN2(num, ovflw_stk->length());
1.3787 + for (int i = (int) num; i > 0; i--) {
1.3788 + oop cur = ovflw_stk->pop();
1.3789 + assert(cur != NULL, "Counted wrong?");
1.3790 + work_q->push(cur);
1.3791 + }
1.3792 + return num > 0;
1.3793 +}
1.3794 +
1.3795 +void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
1.3796 + SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
1.3797 + int n_tasks = pst->n_tasks();
1.3798 + // We allow that there may be no tasks to do here because
1.3799 + // we are restarting after a stack overflow.
1.3800 + assert(pst->valid() || n_tasks == 0, "Uninitializd use?");
1.3801 + int nth_task = 0;
1.3802 +
1.3803 + HeapWord* start = sp->bottom();
1.3804 + size_t chunk_size = sp->marking_task_size();
1.3805 + while (!pst->is_task_claimed(/* reference */ nth_task)) {
1.3806 + // Having claimed the nth task in this space,
1.3807 + // compute the chunk that it corresponds to:
1.3808 + MemRegion span = MemRegion(start + nth_task*chunk_size,
1.3809 + start + (nth_task+1)*chunk_size);
1.3810 + // Try and bump the global finger via a CAS;
1.3811 + // note that we need to do the global finger bump
1.3812 + // _before_ taking the intersection below, because
1.3813 + // the task corresponding to that region will be
1.3814 + // deemed done even if the used_region() expands
1.3815 + // because of allocation -- as it almost certainly will
1.3816 + // during start-up while the threads yield in the
1.3817 + // closure below.
1.3818 + HeapWord* finger = span.end();
1.3819 + bump_global_finger(finger); // atomically
1.3820 + // There are null tasks here corresponding to chunks
1.3821 + // beyond the "top" address of the space.
1.3822 + span = span.intersection(sp->used_region());
1.3823 + if (!span.is_empty()) { // Non-null task
1.3824 + // We want to skip the first object because
1.3825 + // the protocol is to scan any object in its entirety
1.3826 + // that _starts_ in this span; a fortiori, any
1.3827 + // object starting in an earlier span is scanned
1.3828 + // as part of an earlier claimed task.
1.3829 + // Below we use the "careful" version of block_start
1.3830 + // so we do not try to navigate uninitialized objects.
1.3831 + HeapWord* prev_obj = sp->block_start_careful(span.start());
1.3832 + // Below we use a variant of block_size that uses the
1.3833 + // Printezis bits to avoid waiting for allocated
1.3834 + // objects to become initialized/parsable.
1.3835 + while (prev_obj < span.start()) {
1.3836 + size_t sz = sp->block_size_no_stall(prev_obj, _collector);
1.3837 + if (sz > 0) {
1.3838 + prev_obj += sz;
1.3839 + } else {
1.3840 + // In this case we may end up doing a bit of redundant
1.3841 + // scanning, but that appears unavoidable, short of
1.3842 + // locking the free list locks; see bug 6324141.
1.3843 + break;
1.3844 + }
1.3845 + }
1.3846 + if (prev_obj < span.end()) {
1.3847 + MemRegion my_span = MemRegion(prev_obj, span.end());
1.3848 + // Do the marking work within a non-empty span --
1.3849 + // the last argument to the constructor indicates whether the
1.3850 + // iteration should be incremental with periodic yields.
1.3851 + Par_MarkFromRootsClosure cl(this, _collector, my_span,
1.3852 + &_collector->_markBitMap,
1.3853 + work_queue(i),
1.3854 + &_collector->_markStack,
1.3855 + &_collector->_revisitStack,
1.3856 + _asynch);
1.3857 + _collector->_markBitMap.iterate(&cl, my_span.start(), my_span.end());
1.3858 + } // else nothing to do for this task
1.3859 + } // else nothing to do for this task
1.3860 + }
1.3861 + // We'd be tempted to assert here that since there are no
1.3862 + // more tasks left to claim in this space, the global_finger
1.3863 + // must exceed space->top() and a fortiori space->end(). However,
1.3864 + // that would not quite be correct because the bumping of
1.3865 + // global_finger occurs strictly after the claiming of a task,
1.3866 + // so by the time we reach here the global finger may not yet
1.3867 + // have been bumped up by the thread that claimed the last
1.3868 + // task.
1.3869 + pst->all_tasks_completed();
1.3870 +}
1.3871 +
1.3872 +class Par_ConcMarkingClosure: public OopClosure {
1.3873 + CMSCollector* _collector;
1.3874 + MemRegion _span;
1.3875 + CMSBitMap* _bit_map;
1.3876 + CMSMarkStack* _overflow_stack;
1.3877 + CMSMarkStack* _revisit_stack; // XXXXXX Check proper use
1.3878 + OopTaskQueue* _work_queue;
1.3879 +
1.3880 + public:
1.3881 + Par_ConcMarkingClosure(CMSCollector* collector, OopTaskQueue* work_queue,
1.3882 + CMSBitMap* bit_map, CMSMarkStack* overflow_stack):
1.3883 + _collector(collector),
1.3884 + _span(_collector->_span),
1.3885 + _work_queue(work_queue),
1.3886 + _bit_map(bit_map),
1.3887 + _overflow_stack(overflow_stack) { } // need to initialize revisit stack etc.
1.3888 +
1.3889 + void do_oop(oop* p);
1.3890 + void trim_queue(size_t max);
1.3891 + void handle_stack_overflow(HeapWord* lost);
1.3892 +};
1.3893 +
1.3894 +// Grey object rescan during work stealing phase --
1.3895 +// the salient assumption here is that stolen oops must
1.3896 +// always be initialized, so we do not need to check for
1.3897 +// uninitialized objects before scanning here.
1.3898 +void Par_ConcMarkingClosure::do_oop(oop* p) {
1.3899 + oop this_oop = *p;
1.3900 + assert(this_oop->is_oop_or_null(),
1.3901 + "expected an oop or NULL");
1.3902 + HeapWord* addr = (HeapWord*)this_oop;
1.3903 + // Check if oop points into the CMS generation
1.3904 + // and is not marked
1.3905 + if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
1.3906 + // a white object ...
1.3907 + // If we manage to "claim" the object, by being the
1.3908 + // first thread to mark it, then we push it on our
1.3909 + // marking stack
1.3910 + if (_bit_map->par_mark(addr)) { // ... now grey
1.3911 + // push on work queue (grey set)
1.3912 + bool simulate_overflow = false;
1.3913 + NOT_PRODUCT(
1.3914 + if (CMSMarkStackOverflowALot &&
1.3915 + _collector->simulate_overflow()) {
1.3916 + // simulate a stack overflow
1.3917 + simulate_overflow = true;
1.3918 + }
1.3919 + )
1.3920 + if (simulate_overflow ||
1.3921 + !(_work_queue->push(this_oop) || _overflow_stack->par_push(this_oop))) {
1.3922 + // stack overflow
1.3923 + if (PrintCMSStatistics != 0) {
1.3924 + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
1.3925 + SIZE_FORMAT, _overflow_stack->capacity());
1.3926 + }
1.3927 + // We cannot assert that the overflow stack is full because
1.3928 + // it may have been emptied since.
1.3929 + assert(simulate_overflow ||
1.3930 + _work_queue->size() == _work_queue->max_elems(),
1.3931 + "Else push should have succeeded");
1.3932 + handle_stack_overflow(addr);
1.3933 + }
1.3934 + } // Else, some other thread got there first
1.3935 + }
1.3936 +}
1.3937 +
1.3938 +void Par_ConcMarkingClosure::trim_queue(size_t max) {
1.3939 + while (_work_queue->size() > max) {
1.3940 + oop new_oop;
1.3941 + if (_work_queue->pop_local(new_oop)) {
1.3942 + assert(new_oop->is_oop(), "Should be an oop");
1.3943 + assert(_bit_map->isMarked((HeapWord*)new_oop), "Grey object");
1.3944 + assert(_span.contains((HeapWord*)new_oop), "Not in span");
1.3945 + assert(new_oop->is_parsable(), "Should be parsable");
1.3946 + new_oop->oop_iterate(this); // do_oop() above
1.3947 + }
1.3948 + }
1.3949 +}
1.3950 +
1.3951 +// Upon stack overflow, we discard (part of) the stack,
1.3952 +// remembering the least address amongst those discarded
1.3953 +// in CMSCollector's _restart_address.
1.3954 +void Par_ConcMarkingClosure::handle_stack_overflow(HeapWord* lost) {
1.3955 + // We need to do this under a mutex to prevent other
1.3956 + // workers from interfering with the expansion below.
1.3957 + MutexLockerEx ml(_overflow_stack->par_lock(),
1.3958 + Mutex::_no_safepoint_check_flag);
1.3959 + // Remember the least grey address discarded
1.3960 + HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost);
1.3961 + _collector->lower_restart_addr(ra);
1.3962 + _overflow_stack->reset(); // discard stack contents
1.3963 + _overflow_stack->expand(); // expand the stack if possible
1.3964 +}
1.3965 +
1.3966 +
1.3967 +void CMSConcMarkingTask::do_work_steal(int i) {
1.3968 + OopTaskQueue* work_q = work_queue(i);
1.3969 + oop obj_to_scan;
1.3970 + CMSBitMap* bm = &(_collector->_markBitMap);
1.3971 + CMSMarkStack* ovflw = &(_collector->_markStack);
1.3972 + int* seed = _collector->hash_seed(i);
1.3973 + Par_ConcMarkingClosure cl(_collector, work_q, bm, ovflw);
1.3974 + while (true) {
1.3975 + cl.trim_queue(0);
1.3976 + assert(work_q->size() == 0, "Should have been emptied above");
1.3977 + if (get_work_from_overflow_stack(ovflw, work_q)) {
1.3978 + // Can't assert below because the work obtained from the
1.3979 + // overflow stack may already have been stolen from us.
1.3980 + // assert(work_q->size() > 0, "Work from overflow stack");
1.3981 + continue;
1.3982 + } else if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
1.3983 + assert(obj_to_scan->is_oop(), "Should be an oop");
1.3984 + assert(bm->isMarked((HeapWord*)obj_to_scan), "Grey object");
1.3985 + obj_to_scan->oop_iterate(&cl);
1.3986 + } else if (terminator()->offer_termination()) {
1.3987 + assert(work_q->size() == 0, "Impossible!");
1.3988 + break;
1.3989 + }
1.3990 + }
1.3991 +}
1.3992 +
1.3993 +// This is run by the CMS (coordinator) thread.
1.3994 +void CMSConcMarkingTask::coordinator_yield() {
1.3995 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.3996 + "CMS thread should hold CMS token");
1.3997 +
1.3998 + // First give up the locks, then yield, then re-lock
1.3999 + // We should probably use a constructor/destructor idiom to
1.4000 + // do this unlock/lock or modify the MutexUnlocker class to
1.4001 + // serve our purpose. XXX
1.4002 + assert_lock_strong(_bit_map_lock);
1.4003 + _bit_map_lock->unlock();
1.4004 + ConcurrentMarkSweepThread::desynchronize(true);
1.4005 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.4006 + _collector->stopTimer();
1.4007 + if (PrintCMSStatistics != 0) {
1.4008 + _collector->incrementYields();
1.4009 + }
1.4010 + _collector->icms_wait();
1.4011 +
1.4012 + // It is possible for whichever thread initiated the yield request
1.4013 + // not to get a chance to wake up and take the bitmap lock between
1.4014 + // this thread releasing it and reacquiring it. So, while the
1.4015 + // should_yield() flag is on, let's sleep for a bit to give the
1.4016 + // other thread a chance to wake up. The limit imposed on the number
1.4017 + // of iterations is defensive, to avoid any unforseen circumstances
1.4018 + // putting us into an infinite loop. Since it's always been this
1.4019 + // (coordinator_yield()) method that was observed to cause the
1.4020 + // problem, we are using a parameter (CMSCoordinatorYieldSleepCount)
1.4021 + // which is by default non-zero. For the other seven methods that
1.4022 + // also perform the yield operation, as are using a different
1.4023 + // parameter (CMSYieldSleepCount) which is by default zero. This way we
1.4024 + // can enable the sleeping for those methods too, if necessary.
1.4025 + // See 6442774.
1.4026 + //
1.4027 + // We really need to reconsider the synchronization between the GC
1.4028 + // thread and the yield-requesting threads in the future and we
1.4029 + // should really use wait/notify, which is the recommended
1.4030 + // way of doing this type of interaction. Additionally, we should
1.4031 + // consolidate the eight methods that do the yield operation and they
1.4032 + // are almost identical into one for better maintenability and
1.4033 + // readability. See 6445193.
1.4034 + //
1.4035 + // Tony 2006.06.29
1.4036 + for (unsigned i = 0; i < CMSCoordinatorYieldSleepCount &&
1.4037 + ConcurrentMarkSweepThread::should_yield() &&
1.4038 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.4039 + os::sleep(Thread::current(), 1, false);
1.4040 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.4041 + }
1.4042 +
1.4043 + ConcurrentMarkSweepThread::synchronize(true);
1.4044 + _bit_map_lock->lock_without_safepoint_check();
1.4045 + _collector->startTimer();
1.4046 +}
1.4047 +
1.4048 +bool CMSCollector::do_marking_mt(bool asynch) {
1.4049 + assert(ParallelCMSThreads > 0 && conc_workers() != NULL, "precondition");
1.4050 + // In the future this would be determined ergonomically, based
1.4051 + // on #cpu's, # active mutator threads (and load), and mutation rate.
1.4052 + int num_workers = ParallelCMSThreads;
1.4053 +
1.4054 + CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
1.4055 + CompactibleFreeListSpace* perm_space = _permGen->cmsSpace();
1.4056 +
1.4057 + CMSConcMarkingTask tsk(this, cms_space, perm_space,
1.4058 + asynch, num_workers /* number requested XXX */,
1.4059 + conc_workers(), task_queues());
1.4060 +
1.4061 + // Since the actual number of workers we get may be different
1.4062 + // from the number we requested above, do we need to do anything different
1.4063 + // below? In particular, may be we need to subclass the SequantialSubTasksDone
1.4064 + // class?? XXX
1.4065 + cms_space ->initialize_sequential_subtasks_for_marking(num_workers);
1.4066 + perm_space->initialize_sequential_subtasks_for_marking(num_workers);
1.4067 +
1.4068 + // Refs discovery is already non-atomic.
1.4069 + assert(!ref_processor()->discovery_is_atomic(), "Should be non-atomic");
1.4070 + // Mutate the Refs discovery so it is MT during the
1.4071 + // multi-threaded marking phase.
1.4072 + ReferenceProcessorMTMutator mt(ref_processor(), num_workers > 1);
1.4073 +
1.4074 + conc_workers()->start_task(&tsk);
1.4075 + while (tsk.yielded()) {
1.4076 + tsk.coordinator_yield();
1.4077 + conc_workers()->continue_task(&tsk);
1.4078 + }
1.4079 + // If the task was aborted, _restart_addr will be non-NULL
1.4080 + assert(tsk.completed() || _restart_addr != NULL, "Inconsistency");
1.4081 + while (_restart_addr != NULL) {
1.4082 + // XXX For now we do not make use of ABORTED state and have not
1.4083 + // yet implemented the right abort semantics (even in the original
1.4084 + // single-threaded CMS case). That needs some more investigation
1.4085 + // and is deferred for now; see CR# TBF. 07252005YSR. XXX
1.4086 + assert(!CMSAbortSemantics || tsk.aborted(), "Inconsistency");
1.4087 + // If _restart_addr is non-NULL, a marking stack overflow
1.4088 + // occured; we need to do a fresh marking iteration from the
1.4089 + // indicated restart address.
1.4090 + if (_foregroundGCIsActive && asynch) {
1.4091 + // We may be running into repeated stack overflows, having
1.4092 + // reached the limit of the stack size, while making very
1.4093 + // slow forward progress. It may be best to bail out and
1.4094 + // let the foreground collector do its job.
1.4095 + // Clear _restart_addr, so that foreground GC
1.4096 + // works from scratch. This avoids the headache of
1.4097 + // a "rescan" which would otherwise be needed because
1.4098 + // of the dirty mod union table & card table.
1.4099 + _restart_addr = NULL;
1.4100 + return false;
1.4101 + }
1.4102 + // Adjust the task to restart from _restart_addr
1.4103 + tsk.reset(_restart_addr);
1.4104 + cms_space ->initialize_sequential_subtasks_for_marking(num_workers,
1.4105 + _restart_addr);
1.4106 + perm_space->initialize_sequential_subtasks_for_marking(num_workers,
1.4107 + _restart_addr);
1.4108 + _restart_addr = NULL;
1.4109 + // Get the workers going again
1.4110 + conc_workers()->start_task(&tsk);
1.4111 + while (tsk.yielded()) {
1.4112 + tsk.coordinator_yield();
1.4113 + conc_workers()->continue_task(&tsk);
1.4114 + }
1.4115 + }
1.4116 + assert(tsk.completed(), "Inconsistency");
1.4117 + assert(tsk.result() == true, "Inconsistency");
1.4118 + return true;
1.4119 +}
1.4120 +
1.4121 +bool CMSCollector::do_marking_st(bool asynch) {
1.4122 + ResourceMark rm;
1.4123 + HandleMark hm;
1.4124 +
1.4125 + MarkFromRootsClosure markFromRootsClosure(this, _span, &_markBitMap,
1.4126 + &_markStack, &_revisitStack, CMSYield && asynch);
1.4127 + // the last argument to iterate indicates whether the iteration
1.4128 + // should be incremental with periodic yields.
1.4129 + _markBitMap.iterate(&markFromRootsClosure);
1.4130 + // If _restart_addr is non-NULL, a marking stack overflow
1.4131 + // occured; we need to do a fresh iteration from the
1.4132 + // indicated restart address.
1.4133 + while (_restart_addr != NULL) {
1.4134 + if (_foregroundGCIsActive && asynch) {
1.4135 + // We may be running into repeated stack overflows, having
1.4136 + // reached the limit of the stack size, while making very
1.4137 + // slow forward progress. It may be best to bail out and
1.4138 + // let the foreground collector do its job.
1.4139 + // Clear _restart_addr, so that foreground GC
1.4140 + // works from scratch. This avoids the headache of
1.4141 + // a "rescan" which would otherwise be needed because
1.4142 + // of the dirty mod union table & card table.
1.4143 + _restart_addr = NULL;
1.4144 + return false; // indicating failure to complete marking
1.4145 + }
1.4146 + // Deal with stack overflow:
1.4147 + // we restart marking from _restart_addr
1.4148 + HeapWord* ra = _restart_addr;
1.4149 + markFromRootsClosure.reset(ra);
1.4150 + _restart_addr = NULL;
1.4151 + _markBitMap.iterate(&markFromRootsClosure, ra, _span.end());
1.4152 + }
1.4153 + return true;
1.4154 +}
1.4155 +
1.4156 +void CMSCollector::preclean() {
1.4157 + check_correct_thread_executing();
1.4158 + assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
1.4159 + verify_work_stacks_empty();
1.4160 + verify_overflow_empty();
1.4161 + _abort_preclean = false;
1.4162 + if (CMSPrecleaningEnabled) {
1.4163 + _eden_chunk_index = 0;
1.4164 + size_t used = get_eden_used();
1.4165 + size_t capacity = get_eden_capacity();
1.4166 + // Don't start sampling unless we will get sufficiently
1.4167 + // many samples.
1.4168 + if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
1.4169 + * CMSScheduleRemarkEdenPenetration)) {
1.4170 + _start_sampling = true;
1.4171 + } else {
1.4172 + _start_sampling = false;
1.4173 + }
1.4174 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.4175 + CMSPhaseAccounting pa(this, "preclean", !PrintGCDetails);
1.4176 + preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
1.4177 + }
1.4178 + CMSTokenSync x(true); // is cms thread
1.4179 + if (CMSPrecleaningEnabled) {
1.4180 + sample_eden();
1.4181 + _collectorState = AbortablePreclean;
1.4182 + } else {
1.4183 + _collectorState = FinalMarking;
1.4184 + }
1.4185 + verify_work_stacks_empty();
1.4186 + verify_overflow_empty();
1.4187 +}
1.4188 +
1.4189 +// Try and schedule the remark such that young gen
1.4190 +// occupancy is CMSScheduleRemarkEdenPenetration %.
1.4191 +void CMSCollector::abortable_preclean() {
1.4192 + check_correct_thread_executing();
1.4193 + assert(CMSPrecleaningEnabled, "Inconsistent control state");
1.4194 + assert(_collectorState == AbortablePreclean, "Inconsistent control state");
1.4195 +
1.4196 + // If Eden's current occupancy is below this threshold,
1.4197 + // immediately schedule the remark; else preclean
1.4198 + // past the next scavenge in an effort to
1.4199 + // schedule the pause as described avove. By choosing
1.4200 + // CMSScheduleRemarkEdenSizeThreshold >= max eden size
1.4201 + // we will never do an actual abortable preclean cycle.
1.4202 + if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) {
1.4203 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.4204 + CMSPhaseAccounting pa(this, "abortable-preclean", !PrintGCDetails);
1.4205 + // We need more smarts in the abortable preclean
1.4206 + // loop below to deal with cases where allocation
1.4207 + // in young gen is very very slow, and our precleaning
1.4208 + // is running a losing race against a horde of
1.4209 + // mutators intent on flooding us with CMS updates
1.4210 + // (dirty cards).
1.4211 + // One, admittedly dumb, strategy is to give up
1.4212 + // after a certain number of abortable precleaning loops
1.4213 + // or after a certain maximum time. We want to make
1.4214 + // this smarter in the next iteration.
1.4215 + // XXX FIX ME!!! YSR
1.4216 + size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0;
1.4217 + while (!(should_abort_preclean() ||
1.4218 + ConcurrentMarkSweepThread::should_terminate())) {
1.4219 + workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2);
1.4220 + cumworkdone += workdone;
1.4221 + loops++;
1.4222 + // Voluntarily terminate abortable preclean phase if we have
1.4223 + // been at it for too long.
1.4224 + if ((CMSMaxAbortablePrecleanLoops != 0) &&
1.4225 + loops >= CMSMaxAbortablePrecleanLoops) {
1.4226 + if (PrintGCDetails) {
1.4227 + gclog_or_tty->print(" CMS: abort preclean due to loops ");
1.4228 + }
1.4229 + break;
1.4230 + }
1.4231 + if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) {
1.4232 + if (PrintGCDetails) {
1.4233 + gclog_or_tty->print(" CMS: abort preclean due to time ");
1.4234 + }
1.4235 + break;
1.4236 + }
1.4237 + // If we are doing little work each iteration, we should
1.4238 + // take a short break.
1.4239 + if (workdone < CMSAbortablePrecleanMinWorkPerIteration) {
1.4240 + // Sleep for some time, waiting for work to accumulate
1.4241 + stopTimer();
1.4242 + cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis);
1.4243 + startTimer();
1.4244 + waited++;
1.4245 + }
1.4246 + }
1.4247 + if (PrintCMSStatistics > 0) {
1.4248 + gclog_or_tty->print(" [%d iterations, %d waits, %d cards)] ",
1.4249 + loops, waited, cumworkdone);
1.4250 + }
1.4251 + }
1.4252 + CMSTokenSync x(true); // is cms thread
1.4253 + if (_collectorState != Idling) {
1.4254 + assert(_collectorState == AbortablePreclean,
1.4255 + "Spontaneous state transition?");
1.4256 + _collectorState = FinalMarking;
1.4257 + } // Else, a foreground collection completed this CMS cycle.
1.4258 + return;
1.4259 +}
1.4260 +
1.4261 +// Respond to an Eden sampling opportunity
1.4262 +void CMSCollector::sample_eden() {
1.4263 + // Make sure a young gc cannot sneak in between our
1.4264 + // reading and recording of a sample.
1.4265 + assert(Thread::current()->is_ConcurrentGC_thread(),
1.4266 + "Only the cms thread may collect Eden samples");
1.4267 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.4268 + "Should collect samples while holding CMS token");
1.4269 + if (!_start_sampling) {
1.4270 + return;
1.4271 + }
1.4272 + if (_eden_chunk_array) {
1.4273 + if (_eden_chunk_index < _eden_chunk_capacity) {
1.4274 + _eden_chunk_array[_eden_chunk_index] = *_top_addr; // take sample
1.4275 + assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
1.4276 + "Unexpected state of Eden");
1.4277 + // We'd like to check that what we just sampled is an oop-start address;
1.4278 + // however, we cannot do that here since the object may not yet have been
1.4279 + // initialized. So we'll instead do the check when we _use_ this sample
1.4280 + // later.
1.4281 + if (_eden_chunk_index == 0 ||
1.4282 + (pointer_delta(_eden_chunk_array[_eden_chunk_index],
1.4283 + _eden_chunk_array[_eden_chunk_index-1])
1.4284 + >= CMSSamplingGrain)) {
1.4285 + _eden_chunk_index++; // commit sample
1.4286 + }
1.4287 + }
1.4288 + }
1.4289 + if ((_collectorState == AbortablePreclean) && !_abort_preclean) {
1.4290 + size_t used = get_eden_used();
1.4291 + size_t capacity = get_eden_capacity();
1.4292 + assert(used <= capacity, "Unexpected state of Eden");
1.4293 + if (used > (capacity/100 * CMSScheduleRemarkEdenPenetration)) {
1.4294 + _abort_preclean = true;
1.4295 + }
1.4296 + }
1.4297 +}
1.4298 +
1.4299 +
1.4300 +size_t CMSCollector::preclean_work(bool clean_refs, bool clean_survivor) {
1.4301 + assert(_collectorState == Precleaning ||
1.4302 + _collectorState == AbortablePreclean, "incorrect state");
1.4303 + ResourceMark rm;
1.4304 + HandleMark hm;
1.4305 + // Do one pass of scrubbing the discovered reference lists
1.4306 + // to remove any reference objects with strongly-reachable
1.4307 + // referents.
1.4308 + if (clean_refs) {
1.4309 + ReferenceProcessor* rp = ref_processor();
1.4310 + CMSPrecleanRefsYieldClosure yield_cl(this);
1.4311 + assert(rp->span().equals(_span), "Spans should be equal");
1.4312 + CMSKeepAliveClosure keep_alive(this, _span, &_markBitMap,
1.4313 + &_markStack);
1.4314 + CMSDrainMarkingStackClosure complete_trace(this,
1.4315 + _span, &_markBitMap, &_markStack,
1.4316 + &keep_alive);
1.4317 +
1.4318 + // We don't want this step to interfere with a young
1.4319 + // collection because we don't want to take CPU
1.4320 + // or memory bandwidth away from the young GC threads
1.4321 + // (which may be as many as there are CPUs).
1.4322 + // Note that we don't need to protect ourselves from
1.4323 + // interference with mutators because they can't
1.4324 + // manipulate the discovered reference lists nor affect
1.4325 + // the computed reachability of the referents, the
1.4326 + // only properties manipulated by the precleaning
1.4327 + // of these reference lists.
1.4328 + stopTimer();
1.4329 + CMSTokenSyncWithLocks x(true /* is cms thread */,
1.4330 + bitMapLock());
1.4331 + startTimer();
1.4332 + sample_eden();
1.4333 + // The following will yield to allow foreground
1.4334 + // collection to proceed promptly. XXX YSR:
1.4335 + // The code in this method may need further
1.4336 + // tweaking for better performance and some restructuring
1.4337 + // for cleaner interfaces.
1.4338 + rp->preclean_discovered_references(
1.4339 + rp->is_alive_non_header(), &keep_alive, &complete_trace,
1.4340 + &yield_cl);
1.4341 + }
1.4342 +
1.4343 + if (clean_survivor) { // preclean the active survivor space(s)
1.4344 + assert(_young_gen->kind() == Generation::DefNew ||
1.4345 + _young_gen->kind() == Generation::ParNew ||
1.4346 + _young_gen->kind() == Generation::ASParNew,
1.4347 + "incorrect type for cast");
1.4348 + DefNewGeneration* dng = (DefNewGeneration*)_young_gen;
1.4349 + PushAndMarkClosure pam_cl(this, _span, ref_processor(),
1.4350 + &_markBitMap, &_modUnionTable,
1.4351 + &_markStack, &_revisitStack,
1.4352 + true /* precleaning phase */);
1.4353 + stopTimer();
1.4354 + CMSTokenSyncWithLocks ts(true /* is cms thread */,
1.4355 + bitMapLock());
1.4356 + startTimer();
1.4357 + unsigned int before_count =
1.4358 + GenCollectedHeap::heap()->total_collections();
1.4359 + SurvivorSpacePrecleanClosure
1.4360 + sss_cl(this, _span, &_markBitMap, &_markStack,
1.4361 + &pam_cl, before_count, CMSYield);
1.4362 + dng->from()->object_iterate_careful(&sss_cl);
1.4363 + dng->to()->object_iterate_careful(&sss_cl);
1.4364 + }
1.4365 + MarkRefsIntoAndScanClosure
1.4366 + mrias_cl(_span, ref_processor(), &_markBitMap, &_modUnionTable,
1.4367 + &_markStack, &_revisitStack, this, CMSYield,
1.4368 + true /* precleaning phase */);
1.4369 + // CAUTION: The following closure has persistent state that may need to
1.4370 + // be reset upon a decrease in the sequence of addresses it
1.4371 + // processes.
1.4372 + ScanMarkedObjectsAgainCarefullyClosure
1.4373 + smoac_cl(this, _span,
1.4374 + &_markBitMap, &_markStack, &_revisitStack, &mrias_cl, CMSYield);
1.4375 +
1.4376 + // Preclean dirty cards in ModUnionTable and CardTable using
1.4377 + // appropriate convergence criterion;
1.4378 + // repeat CMSPrecleanIter times unless we find that
1.4379 + // we are losing.
1.4380 + assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large");
1.4381 + assert(CMSPrecleanNumerator < CMSPrecleanDenominator,
1.4382 + "Bad convergence multiplier");
1.4383 + assert(CMSPrecleanThreshold >= 100,
1.4384 + "Unreasonably low CMSPrecleanThreshold");
1.4385 +
1.4386 + size_t numIter, cumNumCards, lastNumCards, curNumCards;
1.4387 + for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0;
1.4388 + numIter < CMSPrecleanIter;
1.4389 + numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) {
1.4390 + curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl);
1.4391 + if (CMSPermGenPrecleaningEnabled) {
1.4392 + curNumCards += preclean_mod_union_table(_permGen, &smoac_cl);
1.4393 + }
1.4394 + if (Verbose && PrintGCDetails) {
1.4395 + gclog_or_tty->print(" (modUnionTable: %d cards)", curNumCards);
1.4396 + }
1.4397 + // Either there are very few dirty cards, so re-mark
1.4398 + // pause will be small anyway, or our pre-cleaning isn't
1.4399 + // that much faster than the rate at which cards are being
1.4400 + // dirtied, so we might as well stop and re-mark since
1.4401 + // precleaning won't improve our re-mark time by much.
1.4402 + if (curNumCards <= CMSPrecleanThreshold ||
1.4403 + (numIter > 0 &&
1.4404 + (curNumCards * CMSPrecleanDenominator >
1.4405 + lastNumCards * CMSPrecleanNumerator))) {
1.4406 + numIter++;
1.4407 + cumNumCards += curNumCards;
1.4408 + break;
1.4409 + }
1.4410 + }
1.4411 + curNumCards = preclean_card_table(_cmsGen, &smoac_cl);
1.4412 + if (CMSPermGenPrecleaningEnabled) {
1.4413 + curNumCards += preclean_card_table(_permGen, &smoac_cl);
1.4414 + }
1.4415 + cumNumCards += curNumCards;
1.4416 + if (PrintGCDetails && PrintCMSStatistics != 0) {
1.4417 + gclog_or_tty->print_cr(" (cardTable: %d cards, re-scanned %d cards, %d iterations)",
1.4418 + curNumCards, cumNumCards, numIter);
1.4419 + }
1.4420 + return cumNumCards; // as a measure of useful work done
1.4421 +}
1.4422 +
1.4423 +// PRECLEANING NOTES:
1.4424 +// Precleaning involves:
1.4425 +// . reading the bits of the modUnionTable and clearing the set bits.
1.4426 +// . For the cards corresponding to the set bits, we scan the
1.4427 +// objects on those cards. This means we need the free_list_lock
1.4428 +// so that we can safely iterate over the CMS space when scanning
1.4429 +// for oops.
1.4430 +// . When we scan the objects, we'll be both reading and setting
1.4431 +// marks in the marking bit map, so we'll need the marking bit map.
1.4432 +// . For protecting _collector_state transitions, we take the CGC_lock.
1.4433 +// Note that any races in the reading of of card table entries by the
1.4434 +// CMS thread on the one hand and the clearing of those entries by the
1.4435 +// VM thread or the setting of those entries by the mutator threads on the
1.4436 +// other are quite benign. However, for efficiency it makes sense to keep
1.4437 +// the VM thread from racing with the CMS thread while the latter is
1.4438 +// dirty card info to the modUnionTable. We therefore also use the
1.4439 +// CGC_lock to protect the reading of the card table and the mod union
1.4440 +// table by the CM thread.
1.4441 +// . We run concurrently with mutator updates, so scanning
1.4442 +// needs to be done carefully -- we should not try to scan
1.4443 +// potentially uninitialized objects.
1.4444 +//
1.4445 +// Locking strategy: While holding the CGC_lock, we scan over and
1.4446 +// reset a maximal dirty range of the mod union / card tables, then lock
1.4447 +// the free_list_lock and bitmap lock to do a full marking, then
1.4448 +// release these locks; and repeat the cycle. This allows for a
1.4449 +// certain amount of fairness in the sharing of these locks between
1.4450 +// the CMS collector on the one hand, and the VM thread and the
1.4451 +// mutators on the other.
1.4452 +
1.4453 +// NOTE: preclean_mod_union_table() and preclean_card_table()
1.4454 +// further below are largely identical; if you need to modify
1.4455 +// one of these methods, please check the other method too.
1.4456 +
1.4457 +size_t CMSCollector::preclean_mod_union_table(
1.4458 + ConcurrentMarkSweepGeneration* gen,
1.4459 + ScanMarkedObjectsAgainCarefullyClosure* cl) {
1.4460 + verify_work_stacks_empty();
1.4461 + verify_overflow_empty();
1.4462 +
1.4463 + // strategy: starting with the first card, accumulate contiguous
1.4464 + // ranges of dirty cards; clear these cards, then scan the region
1.4465 + // covered by these cards.
1.4466 +
1.4467 + // Since all of the MUT is committed ahead, we can just use
1.4468 + // that, in case the generations expand while we are precleaning.
1.4469 + // It might also be fine to just use the committed part of the
1.4470 + // generation, but we might potentially miss cards when the
1.4471 + // generation is rapidly expanding while we are in the midst
1.4472 + // of precleaning.
1.4473 + HeapWord* startAddr = gen->reserved().start();
1.4474 + HeapWord* endAddr = gen->reserved().end();
1.4475 +
1.4476 + cl->setFreelistLock(gen->freelistLock()); // needed for yielding
1.4477 +
1.4478 + size_t numDirtyCards, cumNumDirtyCards;
1.4479 + HeapWord *nextAddr, *lastAddr;
1.4480 + for (cumNumDirtyCards = numDirtyCards = 0,
1.4481 + nextAddr = lastAddr = startAddr;
1.4482 + nextAddr < endAddr;
1.4483 + nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
1.4484 +
1.4485 + ResourceMark rm;
1.4486 + HandleMark hm;
1.4487 +
1.4488 + MemRegion dirtyRegion;
1.4489 + {
1.4490 + stopTimer();
1.4491 + CMSTokenSync ts(true);
1.4492 + startTimer();
1.4493 + sample_eden();
1.4494 + // Get dirty region starting at nextOffset (inclusive),
1.4495 + // simultaneously clearing it.
1.4496 + dirtyRegion =
1.4497 + _modUnionTable.getAndClearMarkedRegion(nextAddr, endAddr);
1.4498 + assert(dirtyRegion.start() >= nextAddr,
1.4499 + "returned region inconsistent?");
1.4500 + }
1.4501 + // Remember where the next search should begin.
1.4502 + // The returned region (if non-empty) is a right open interval,
1.4503 + // so lastOffset is obtained from the right end of that
1.4504 + // interval.
1.4505 + lastAddr = dirtyRegion.end();
1.4506 + // Should do something more transparent and less hacky XXX
1.4507 + numDirtyCards =
1.4508 + _modUnionTable.heapWordDiffToOffsetDiff(dirtyRegion.word_size());
1.4509 +
1.4510 + // We'll scan the cards in the dirty region (with periodic
1.4511 + // yields for foreground GC as needed).
1.4512 + if (!dirtyRegion.is_empty()) {
1.4513 + assert(numDirtyCards > 0, "consistency check");
1.4514 + HeapWord* stop_point = NULL;
1.4515 + {
1.4516 + stopTimer();
1.4517 + CMSTokenSyncWithLocks ts(true, gen->freelistLock(),
1.4518 + bitMapLock());
1.4519 + startTimer();
1.4520 + verify_work_stacks_empty();
1.4521 + verify_overflow_empty();
1.4522 + sample_eden();
1.4523 + stop_point =
1.4524 + gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
1.4525 + }
1.4526 + if (stop_point != NULL) {
1.4527 + // The careful iteration stopped early either because it found an
1.4528 + // uninitialized object, or because we were in the midst of an
1.4529 + // "abortable preclean", which should now be aborted. Redirty
1.4530 + // the bits corresponding to the partially-scanned or unscanned
1.4531 + // cards. We'll either restart at the next block boundary or
1.4532 + // abort the preclean.
1.4533 + assert((CMSPermGenPrecleaningEnabled && (gen == _permGen)) ||
1.4534 + (_collectorState == AbortablePreclean && should_abort_preclean()),
1.4535 + "Unparsable objects should only be in perm gen.");
1.4536 +
1.4537 + stopTimer();
1.4538 + CMSTokenSyncWithLocks ts(true, bitMapLock());
1.4539 + startTimer();
1.4540 + _modUnionTable.mark_range(MemRegion(stop_point, dirtyRegion.end()));
1.4541 + if (should_abort_preclean()) {
1.4542 + break; // out of preclean loop
1.4543 + } else {
1.4544 + // Compute the next address at which preclean should pick up;
1.4545 + // might need bitMapLock in order to read P-bits.
1.4546 + lastAddr = next_card_start_after_block(stop_point);
1.4547 + }
1.4548 + }
1.4549 + } else {
1.4550 + assert(lastAddr == endAddr, "consistency check");
1.4551 + assert(numDirtyCards == 0, "consistency check");
1.4552 + break;
1.4553 + }
1.4554 + }
1.4555 + verify_work_stacks_empty();
1.4556 + verify_overflow_empty();
1.4557 + return cumNumDirtyCards;
1.4558 +}
1.4559 +
1.4560 +// NOTE: preclean_mod_union_table() above and preclean_card_table()
1.4561 +// below are largely identical; if you need to modify
1.4562 +// one of these methods, please check the other method too.
1.4563 +
1.4564 +size_t CMSCollector::preclean_card_table(ConcurrentMarkSweepGeneration* gen,
1.4565 + ScanMarkedObjectsAgainCarefullyClosure* cl) {
1.4566 + // strategy: it's similar to precleamModUnionTable above, in that
1.4567 + // we accumulate contiguous ranges of dirty cards, mark these cards
1.4568 + // precleaned, then scan the region covered by these cards.
1.4569 + HeapWord* endAddr = (HeapWord*)(gen->_virtual_space.high());
1.4570 + HeapWord* startAddr = (HeapWord*)(gen->_virtual_space.low());
1.4571 +
1.4572 + cl->setFreelistLock(gen->freelistLock()); // needed for yielding
1.4573 +
1.4574 + size_t numDirtyCards, cumNumDirtyCards;
1.4575 + HeapWord *lastAddr, *nextAddr;
1.4576 +
1.4577 + for (cumNumDirtyCards = numDirtyCards = 0,
1.4578 + nextAddr = lastAddr = startAddr;
1.4579 + nextAddr < endAddr;
1.4580 + nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
1.4581 +
1.4582 + ResourceMark rm;
1.4583 + HandleMark hm;
1.4584 +
1.4585 + MemRegion dirtyRegion;
1.4586 + {
1.4587 + // See comments in "Precleaning notes" above on why we
1.4588 + // do this locking. XXX Could the locking overheads be
1.4589 + // too high when dirty cards are sparse? [I don't think so.]
1.4590 + stopTimer();
1.4591 + CMSTokenSync x(true); // is cms thread
1.4592 + startTimer();
1.4593 + sample_eden();
1.4594 + // Get and clear dirty region from card table
1.4595 + dirtyRegion = _ct->ct_bs()->dirty_card_range_after_preclean(
1.4596 + MemRegion(nextAddr, endAddr));
1.4597 + assert(dirtyRegion.start() >= nextAddr,
1.4598 + "returned region inconsistent?");
1.4599 + }
1.4600 + lastAddr = dirtyRegion.end();
1.4601 + numDirtyCards =
1.4602 + dirtyRegion.word_size()/CardTableModRefBS::card_size_in_words;
1.4603 +
1.4604 + if (!dirtyRegion.is_empty()) {
1.4605 + stopTimer();
1.4606 + CMSTokenSyncWithLocks ts(true, gen->freelistLock(), bitMapLock());
1.4607 + startTimer();
1.4608 + sample_eden();
1.4609 + verify_work_stacks_empty();
1.4610 + verify_overflow_empty();
1.4611 + HeapWord* stop_point =
1.4612 + gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
1.4613 + if (stop_point != NULL) {
1.4614 + // The careful iteration stopped early because it found an
1.4615 + // uninitialized object. Redirty the bits corresponding to the
1.4616 + // partially-scanned or unscanned cards, and start again at the
1.4617 + // next block boundary.
1.4618 + assert(CMSPermGenPrecleaningEnabled ||
1.4619 + (_collectorState == AbortablePreclean && should_abort_preclean()),
1.4620 + "Unparsable objects should only be in perm gen.");
1.4621 + _ct->ct_bs()->invalidate(MemRegion(stop_point, dirtyRegion.end()));
1.4622 + if (should_abort_preclean()) {
1.4623 + break; // out of preclean loop
1.4624 + } else {
1.4625 + // Compute the next address at which preclean should pick up.
1.4626 + lastAddr = next_card_start_after_block(stop_point);
1.4627 + }
1.4628 + }
1.4629 + } else {
1.4630 + break;
1.4631 + }
1.4632 + }
1.4633 + verify_work_stacks_empty();
1.4634 + verify_overflow_empty();
1.4635 + return cumNumDirtyCards;
1.4636 +}
1.4637 +
1.4638 +void CMSCollector::checkpointRootsFinal(bool asynch,
1.4639 + bool clear_all_soft_refs, bool init_mark_was_synchronous) {
1.4640 + assert(_collectorState == FinalMarking, "incorrect state transition?");
1.4641 + check_correct_thread_executing();
1.4642 + // world is stopped at this checkpoint
1.4643 + assert(SafepointSynchronize::is_at_safepoint(),
1.4644 + "world should be stopped");
1.4645 + verify_work_stacks_empty();
1.4646 + verify_overflow_empty();
1.4647 +
1.4648 + SpecializationStats::clear();
1.4649 + if (PrintGCDetails) {
1.4650 + gclog_or_tty->print("[YG occupancy: "SIZE_FORMAT" K ("SIZE_FORMAT" K)]",
1.4651 + _young_gen->used() / K,
1.4652 + _young_gen->capacity() / K);
1.4653 + }
1.4654 + if (asynch) {
1.4655 + if (CMSScavengeBeforeRemark) {
1.4656 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.4657 + // Temporarily set flag to false, GCH->do_collection will
1.4658 + // expect it to be false and set to true
1.4659 + FlagSetting fl(gch->_is_gc_active, false);
1.4660 + NOT_PRODUCT(TraceTime t("Scavenge-Before-Remark",
1.4661 + PrintGCDetails && Verbose, true, gclog_or_tty);)
1.4662 + int level = _cmsGen->level() - 1;
1.4663 + if (level >= 0) {
1.4664 + gch->do_collection(true, // full (i.e. force, see below)
1.4665 + false, // !clear_all_soft_refs
1.4666 + 0, // size
1.4667 + false, // is_tlab
1.4668 + level // max_level
1.4669 + );
1.4670 + }
1.4671 + }
1.4672 + FreelistLocker x(this);
1.4673 + MutexLockerEx y(bitMapLock(),
1.4674 + Mutex::_no_safepoint_check_flag);
1.4675 + assert(!init_mark_was_synchronous, "but that's impossible!");
1.4676 + checkpointRootsFinalWork(asynch, clear_all_soft_refs, false);
1.4677 + } else {
1.4678 + // already have all the locks
1.4679 + checkpointRootsFinalWork(asynch, clear_all_soft_refs,
1.4680 + init_mark_was_synchronous);
1.4681 + }
1.4682 + verify_work_stacks_empty();
1.4683 + verify_overflow_empty();
1.4684 + SpecializationStats::print();
1.4685 +}
1.4686 +
1.4687 +void CMSCollector::checkpointRootsFinalWork(bool asynch,
1.4688 + bool clear_all_soft_refs, bool init_mark_was_synchronous) {
1.4689 +
1.4690 + NOT_PRODUCT(TraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, gclog_or_tty);)
1.4691 +
1.4692 + assert(haveFreelistLocks(), "must have free list locks");
1.4693 + assert_lock_strong(bitMapLock());
1.4694 +
1.4695 + if (UseAdaptiveSizePolicy) {
1.4696 + size_policy()->checkpoint_roots_final_begin();
1.4697 + }
1.4698 +
1.4699 + ResourceMark rm;
1.4700 + HandleMark hm;
1.4701 +
1.4702 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.4703 +
1.4704 + if (cms_should_unload_classes()) {
1.4705 + CodeCache::gc_prologue();
1.4706 + }
1.4707 + assert(haveFreelistLocks(), "must have free list locks");
1.4708 + assert_lock_strong(bitMapLock());
1.4709 +
1.4710 + if (!init_mark_was_synchronous) {
1.4711 + // We might assume that we need not fill TLAB's when
1.4712 + // CMSScavengeBeforeRemark is set, because we may have just done
1.4713 + // a scavenge which would have filled all TLAB's -- and besides
1.4714 + // Eden would be empty. This however may not always be the case --
1.4715 + // for instance although we asked for a scavenge, it may not have
1.4716 + // happened because of a JNI critical section. We probably need
1.4717 + // a policy for deciding whether we can in that case wait until
1.4718 + // the critical section releases and then do the remark following
1.4719 + // the scavenge, and skip it here. In the absence of that policy,
1.4720 + // or of an indication of whether the scavenge did indeed occur,
1.4721 + // we cannot rely on TLAB's having been filled and must do
1.4722 + // so here just in case a scavenge did not happen.
1.4723 + gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
1.4724 + // Update the saved marks which may affect the root scans.
1.4725 + gch->save_marks();
1.4726 +
1.4727 + {
1.4728 + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
1.4729 +
1.4730 + // Note on the role of the mod union table:
1.4731 + // Since the marker in "markFromRoots" marks concurrently with
1.4732 + // mutators, it is possible for some reachable objects not to have been
1.4733 + // scanned. For instance, an only reference to an object A was
1.4734 + // placed in object B after the marker scanned B. Unless B is rescanned,
1.4735 + // A would be collected. Such updates to references in marked objects
1.4736 + // are detected via the mod union table which is the set of all cards
1.4737 + // dirtied since the first checkpoint in this GC cycle and prior to
1.4738 + // the most recent young generation GC, minus those cleaned up by the
1.4739 + // concurrent precleaning.
1.4740 + if (CMSParallelRemarkEnabled && ParallelGCThreads > 0) {
1.4741 + TraceTime t("Rescan (parallel) ", PrintGCDetails, false, gclog_or_tty);
1.4742 + do_remark_parallel();
1.4743 + } else {
1.4744 + TraceTime t("Rescan (non-parallel) ", PrintGCDetails, false,
1.4745 + gclog_or_tty);
1.4746 + do_remark_non_parallel();
1.4747 + }
1.4748 + }
1.4749 + } else {
1.4750 + assert(!asynch, "Can't have init_mark_was_synchronous in asynch mode");
1.4751 + // The initial mark was stop-world, so there's no rescanning to
1.4752 + // do; go straight on to the next step below.
1.4753 + }
1.4754 + verify_work_stacks_empty();
1.4755 + verify_overflow_empty();
1.4756 +
1.4757 + {
1.4758 + NOT_PRODUCT(TraceTime ts("refProcessingWork", PrintGCDetails, false, gclog_or_tty);)
1.4759 + refProcessingWork(asynch, clear_all_soft_refs);
1.4760 + }
1.4761 + verify_work_stacks_empty();
1.4762 + verify_overflow_empty();
1.4763 +
1.4764 + if (cms_should_unload_classes()) {
1.4765 + CodeCache::gc_epilogue();
1.4766 + }
1.4767 +
1.4768 + // If we encountered any (marking stack / work queue) overflow
1.4769 + // events during the current CMS cycle, take appropriate
1.4770 + // remedial measures, where possible, so as to try and avoid
1.4771 + // recurrence of that condition.
1.4772 + assert(_markStack.isEmpty(), "No grey objects");
1.4773 + size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
1.4774 + _ser_kac_ovflw;
1.4775 + if (ser_ovflw > 0) {
1.4776 + if (PrintCMSStatistics != 0) {
1.4777 + gclog_or_tty->print_cr("Marking stack overflow (benign) "
1.4778 + "(pmc_pc="SIZE_FORMAT", pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT")",
1.4779 + _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw,
1.4780 + _ser_kac_ovflw);
1.4781 + }
1.4782 + _markStack.expand();
1.4783 + _ser_pmc_remark_ovflw = 0;
1.4784 + _ser_pmc_preclean_ovflw = 0;
1.4785 + _ser_kac_ovflw = 0;
1.4786 + }
1.4787 + if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
1.4788 + if (PrintCMSStatistics != 0) {
1.4789 + gclog_or_tty->print_cr("Work queue overflow (benign) "
1.4790 + "(pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT")",
1.4791 + _par_pmc_remark_ovflw, _par_kac_ovflw);
1.4792 + }
1.4793 + _par_pmc_remark_ovflw = 0;
1.4794 + _par_kac_ovflw = 0;
1.4795 + }
1.4796 + if (PrintCMSStatistics != 0) {
1.4797 + if (_markStack._hit_limit > 0) {
1.4798 + gclog_or_tty->print_cr(" (benign) Hit max stack size limit ("SIZE_FORMAT")",
1.4799 + _markStack._hit_limit);
1.4800 + }
1.4801 + if (_markStack._failed_double > 0) {
1.4802 + gclog_or_tty->print_cr(" (benign) Failed stack doubling ("SIZE_FORMAT"),"
1.4803 + " current capacity "SIZE_FORMAT,
1.4804 + _markStack._failed_double,
1.4805 + _markStack.capacity());
1.4806 + }
1.4807 + }
1.4808 + _markStack._hit_limit = 0;
1.4809 + _markStack._failed_double = 0;
1.4810 +
1.4811 + if ((VerifyAfterGC || VerifyDuringGC) &&
1.4812 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.4813 + verify_after_remark();
1.4814 + }
1.4815 +
1.4816 + // Change under the freelistLocks.
1.4817 + _collectorState = Sweeping;
1.4818 + // Call isAllClear() under bitMapLock
1.4819 + assert(_modUnionTable.isAllClear(), "Should be clear by end of the"
1.4820 + " final marking");
1.4821 + if (UseAdaptiveSizePolicy) {
1.4822 + size_policy()->checkpoint_roots_final_end(gch->gc_cause());
1.4823 + }
1.4824 +}
1.4825 +
1.4826 +// Parallel remark task
1.4827 +class CMSParRemarkTask: public AbstractGangTask {
1.4828 + CMSCollector* _collector;
1.4829 + WorkGang* _workers;
1.4830 + int _n_workers;
1.4831 + CompactibleFreeListSpace* _cms_space;
1.4832 + CompactibleFreeListSpace* _perm_space;
1.4833 +
1.4834 + // The per-thread work queues, available here for stealing.
1.4835 + OopTaskQueueSet* _task_queues;
1.4836 + ParallelTaskTerminator _term;
1.4837 +
1.4838 + public:
1.4839 + CMSParRemarkTask(CMSCollector* collector,
1.4840 + CompactibleFreeListSpace* cms_space,
1.4841 + CompactibleFreeListSpace* perm_space,
1.4842 + int n_workers, WorkGang* workers,
1.4843 + OopTaskQueueSet* task_queues):
1.4844 + AbstractGangTask("Rescan roots and grey objects in parallel"),
1.4845 + _collector(collector),
1.4846 + _cms_space(cms_space), _perm_space(perm_space),
1.4847 + _n_workers(n_workers),
1.4848 + _workers(workers),
1.4849 + _task_queues(task_queues),
1.4850 + _term(workers->total_workers(), task_queues) { }
1.4851 +
1.4852 + OopTaskQueueSet* task_queues() { return _task_queues; }
1.4853 +
1.4854 + OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
1.4855 +
1.4856 + ParallelTaskTerminator* terminator() { return &_term; }
1.4857 +
1.4858 + void work(int i);
1.4859 +
1.4860 + private:
1.4861 + // Work method in support of parallel rescan ... of young gen spaces
1.4862 + void do_young_space_rescan(int i, Par_MarkRefsIntoAndScanClosure* cl,
1.4863 + ContiguousSpace* space,
1.4864 + HeapWord** chunk_array, size_t chunk_top);
1.4865 +
1.4866 + // ... of dirty cards in old space
1.4867 + void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i,
1.4868 + Par_MarkRefsIntoAndScanClosure* cl);
1.4869 +
1.4870 + // ... work stealing for the above
1.4871 + void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed);
1.4872 +};
1.4873 +
1.4874 +void CMSParRemarkTask::work(int i) {
1.4875 + elapsedTimer _timer;
1.4876 + ResourceMark rm;
1.4877 + HandleMark hm;
1.4878 +
1.4879 + // ---------- rescan from roots --------------
1.4880 + _timer.start();
1.4881 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.4882 + Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
1.4883 + _collector->_span, _collector->ref_processor(),
1.4884 + &(_collector->_markBitMap),
1.4885 + work_queue(i), &(_collector->_revisitStack));
1.4886 +
1.4887 + // Rescan young gen roots first since these are likely
1.4888 + // coarsely partitioned and may, on that account, constitute
1.4889 + // the critical path; thus, it's best to start off that
1.4890 + // work first.
1.4891 + // ---------- young gen roots --------------
1.4892 + {
1.4893 + DefNewGeneration* dng = _collector->_young_gen->as_DefNewGeneration();
1.4894 + EdenSpace* eden_space = dng->eden();
1.4895 + ContiguousSpace* from_space = dng->from();
1.4896 + ContiguousSpace* to_space = dng->to();
1.4897 +
1.4898 + HeapWord** eca = _collector->_eden_chunk_array;
1.4899 + size_t ect = _collector->_eden_chunk_index;
1.4900 + HeapWord** sca = _collector->_survivor_chunk_array;
1.4901 + size_t sct = _collector->_survivor_chunk_index;
1.4902 +
1.4903 + assert(ect <= _collector->_eden_chunk_capacity, "out of bounds");
1.4904 + assert(sct <= _collector->_survivor_chunk_capacity, "out of bounds");
1.4905 +
1.4906 + do_young_space_rescan(i, &par_mrias_cl, to_space, NULL, 0);
1.4907 + do_young_space_rescan(i, &par_mrias_cl, from_space, sca, sct);
1.4908 + do_young_space_rescan(i, &par_mrias_cl, eden_space, eca, ect);
1.4909 +
1.4910 + _timer.stop();
1.4911 + if (PrintCMSStatistics != 0) {
1.4912 + gclog_or_tty->print_cr(
1.4913 + "Finished young gen rescan work in %dth thread: %3.3f sec",
1.4914 + i, _timer.seconds());
1.4915 + }
1.4916 + }
1.4917 +
1.4918 + // ---------- remaining roots --------------
1.4919 + _timer.reset();
1.4920 + _timer.start();
1.4921 + gch->gen_process_strong_roots(_collector->_cmsGen->level(),
1.4922 + false, // yg was scanned above
1.4923 + true, // collecting perm gen
1.4924 + SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
1.4925 + NULL, &par_mrias_cl);
1.4926 + _timer.stop();
1.4927 + if (PrintCMSStatistics != 0) {
1.4928 + gclog_or_tty->print_cr(
1.4929 + "Finished remaining root rescan work in %dth thread: %3.3f sec",
1.4930 + i, _timer.seconds());
1.4931 + }
1.4932 +
1.4933 + // ---------- rescan dirty cards ------------
1.4934 + _timer.reset();
1.4935 + _timer.start();
1.4936 +
1.4937 + // Do the rescan tasks for each of the two spaces
1.4938 + // (cms_space and perm_space) in turn.
1.4939 + do_dirty_card_rescan_tasks(_cms_space, i, &par_mrias_cl);
1.4940 + do_dirty_card_rescan_tasks(_perm_space, i, &par_mrias_cl);
1.4941 + _timer.stop();
1.4942 + if (PrintCMSStatistics != 0) {
1.4943 + gclog_or_tty->print_cr(
1.4944 + "Finished dirty card rescan work in %dth thread: %3.3f sec",
1.4945 + i, _timer.seconds());
1.4946 + }
1.4947 +
1.4948 + // ---------- steal work from other threads ...
1.4949 + // ---------- ... and drain overflow list.
1.4950 + _timer.reset();
1.4951 + _timer.start();
1.4952 + do_work_steal(i, &par_mrias_cl, _collector->hash_seed(i));
1.4953 + _timer.stop();
1.4954 + if (PrintCMSStatistics != 0) {
1.4955 + gclog_or_tty->print_cr(
1.4956 + "Finished work stealing in %dth thread: %3.3f sec",
1.4957 + i, _timer.seconds());
1.4958 + }
1.4959 +}
1.4960 +
1.4961 +void
1.4962 +CMSParRemarkTask::do_young_space_rescan(int i,
1.4963 + Par_MarkRefsIntoAndScanClosure* cl, ContiguousSpace* space,
1.4964 + HeapWord** chunk_array, size_t chunk_top) {
1.4965 + // Until all tasks completed:
1.4966 + // . claim an unclaimed task
1.4967 + // . compute region boundaries corresponding to task claimed
1.4968 + // using chunk_array
1.4969 + // . par_oop_iterate(cl) over that region
1.4970 +
1.4971 + ResourceMark rm;
1.4972 + HandleMark hm;
1.4973 +
1.4974 + SequentialSubTasksDone* pst = space->par_seq_tasks();
1.4975 + assert(pst->valid(), "Uninitialized use?");
1.4976 +
1.4977 + int nth_task = 0;
1.4978 + int n_tasks = pst->n_tasks();
1.4979 +
1.4980 + HeapWord *start, *end;
1.4981 + while (!pst->is_task_claimed(/* reference */ nth_task)) {
1.4982 + // We claimed task # nth_task; compute its boundaries.
1.4983 + if (chunk_top == 0) { // no samples were taken
1.4984 + assert(nth_task == 0 && n_tasks == 1, "Can have only 1 EdenSpace task");
1.4985 + start = space->bottom();
1.4986 + end = space->top();
1.4987 + } else if (nth_task == 0) {
1.4988 + start = space->bottom();
1.4989 + end = chunk_array[nth_task];
1.4990 + } else if (nth_task < (jint)chunk_top) {
1.4991 + assert(nth_task >= 1, "Control point invariant");
1.4992 + start = chunk_array[nth_task - 1];
1.4993 + end = chunk_array[nth_task];
1.4994 + } else {
1.4995 + assert(nth_task == (jint)chunk_top, "Control point invariant");
1.4996 + start = chunk_array[chunk_top - 1];
1.4997 + end = space->top();
1.4998 + }
1.4999 + MemRegion mr(start, end);
1.5000 + // Verify that mr is in space
1.5001 + assert(mr.is_empty() || space->used_region().contains(mr),
1.5002 + "Should be in space");
1.5003 + // Verify that "start" is an object boundary
1.5004 + assert(mr.is_empty() || oop(mr.start())->is_oop(),
1.5005 + "Should be an oop");
1.5006 + space->par_oop_iterate(mr, cl);
1.5007 + }
1.5008 + pst->all_tasks_completed();
1.5009 +}
1.5010 +
1.5011 +void
1.5012 +CMSParRemarkTask::do_dirty_card_rescan_tasks(
1.5013 + CompactibleFreeListSpace* sp, int i,
1.5014 + Par_MarkRefsIntoAndScanClosure* cl) {
1.5015 + // Until all tasks completed:
1.5016 + // . claim an unclaimed task
1.5017 + // . compute region boundaries corresponding to task claimed
1.5018 + // . transfer dirty bits ct->mut for that region
1.5019 + // . apply rescanclosure to dirty mut bits for that region
1.5020 +
1.5021 + ResourceMark rm;
1.5022 + HandleMark hm;
1.5023 +
1.5024 + OopTaskQueue* work_q = work_queue(i);
1.5025 + ModUnionClosure modUnionClosure(&(_collector->_modUnionTable));
1.5026 + // CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION!
1.5027 + // CAUTION: This closure has state that persists across calls to
1.5028 + // the work method dirty_range_iterate_clear() in that it has
1.5029 + // imbedded in it a (subtype of) UpwardsObjectClosure. The
1.5030 + // use of that state in the imbedded UpwardsObjectClosure instance
1.5031 + // assumes that the cards are always iterated (even if in parallel
1.5032 + // by several threads) in monotonically increasing order per each
1.5033 + // thread. This is true of the implementation below which picks
1.5034 + // card ranges (chunks) in monotonically increasing order globally
1.5035 + // and, a-fortiori, in monotonically increasing order per thread
1.5036 + // (the latter order being a subsequence of the former).
1.5037 + // If the work code below is ever reorganized into a more chaotic
1.5038 + // work-partitioning form than the current "sequential tasks"
1.5039 + // paradigm, the use of that persistent state will have to be
1.5040 + // revisited and modified appropriately. See also related
1.5041 + // bug 4756801 work on which should examine this code to make
1.5042 + // sure that the changes there do not run counter to the
1.5043 + // assumptions made here and necessary for correctness and
1.5044 + // efficiency. Note also that this code might yield inefficient
1.5045 + // behaviour in the case of very large objects that span one or
1.5046 + // more work chunks. Such objects would potentially be scanned
1.5047 + // several times redundantly. Work on 4756801 should try and
1.5048 + // address that performance anomaly if at all possible. XXX
1.5049 + MemRegion full_span = _collector->_span;
1.5050 + CMSBitMap* bm = &(_collector->_markBitMap); // shared
1.5051 + CMSMarkStack* rs = &(_collector->_revisitStack); // shared
1.5052 + MarkFromDirtyCardsClosure
1.5053 + greyRescanClosure(_collector, full_span, // entire span of interest
1.5054 + sp, bm, work_q, rs, cl);
1.5055 +
1.5056 + SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
1.5057 + assert(pst->valid(), "Uninitialized use?");
1.5058 + int nth_task = 0;
1.5059 + const int alignment = CardTableModRefBS::card_size * BitsPerWord;
1.5060 + MemRegion span = sp->used_region();
1.5061 + HeapWord* start_addr = span.start();
1.5062 + HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(),
1.5063 + alignment);
1.5064 + const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units
1.5065 + assert((HeapWord*)round_to((intptr_t)start_addr, alignment) ==
1.5066 + start_addr, "Check alignment");
1.5067 + assert((size_t)round_to((intptr_t)chunk_size, alignment) ==
1.5068 + chunk_size, "Check alignment");
1.5069 +
1.5070 + while (!pst->is_task_claimed(/* reference */ nth_task)) {
1.5071 + // Having claimed the nth_task, compute corresponding mem-region,
1.5072 + // which is a-fortiori aligned correctly (i.e. at a MUT bopundary).
1.5073 + // The alignment restriction ensures that we do not need any
1.5074 + // synchronization with other gang-workers while setting or
1.5075 + // clearing bits in thus chunk of the MUT.
1.5076 + MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size,
1.5077 + start_addr + (nth_task+1)*chunk_size);
1.5078 + // The last chunk's end might be way beyond end of the
1.5079 + // used region. In that case pull back appropriately.
1.5080 + if (this_span.end() > end_addr) {
1.5081 + this_span.set_end(end_addr);
1.5082 + assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)");
1.5083 + }
1.5084 + // Iterate over the dirty cards covering this chunk, marking them
1.5085 + // precleaned, and setting the corresponding bits in the mod union
1.5086 + // table. Since we have been careful to partition at Card and MUT-word
1.5087 + // boundaries no synchronization is needed between parallel threads.
1.5088 + _collector->_ct->ct_bs()->dirty_card_iterate(this_span,
1.5089 + &modUnionClosure);
1.5090 +
1.5091 + // Having transferred these marks into the modUnionTable,
1.5092 + // rescan the marked objects on the dirty cards in the modUnionTable.
1.5093 + // Even if this is at a synchronous collection, the initial marking
1.5094 + // may have been done during an asynchronous collection so there
1.5095 + // may be dirty bits in the mod-union table.
1.5096 + _collector->_modUnionTable.dirty_range_iterate_clear(
1.5097 + this_span, &greyRescanClosure);
1.5098 + _collector->_modUnionTable.verifyNoOneBitsInRange(
1.5099 + this_span.start(),
1.5100 + this_span.end());
1.5101 + }
1.5102 + pst->all_tasks_completed(); // declare that i am done
1.5103 +}
1.5104 +
1.5105 +// . see if we can share work_queues with ParNew? XXX
1.5106 +void
1.5107 +CMSParRemarkTask::do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl,
1.5108 + int* seed) {
1.5109 + OopTaskQueue* work_q = work_queue(i);
1.5110 + NOT_PRODUCT(int num_steals = 0;)
1.5111 + oop obj_to_scan;
1.5112 + CMSBitMap* bm = &(_collector->_markBitMap);
1.5113 + size_t num_from_overflow_list =
1.5114 + MIN2((size_t)work_q->max_elems()/4,
1.5115 + (size_t)ParGCDesiredObjsFromOverflowList);
1.5116 +
1.5117 + while (true) {
1.5118 + // Completely finish any left over work from (an) earlier round(s)
1.5119 + cl->trim_queue(0);
1.5120 + // Now check if there's any work in the overflow list
1.5121 + if (_collector->par_take_from_overflow_list(num_from_overflow_list,
1.5122 + work_q)) {
1.5123 + // found something in global overflow list;
1.5124 + // not yet ready to go stealing work from others.
1.5125 + // We'd like to assert(work_q->size() != 0, ...)
1.5126 + // because we just took work from the overflow list,
1.5127 + // but of course we can't since all of that could have
1.5128 + // been already stolen from us.
1.5129 + // "He giveth and He taketh away."
1.5130 + continue;
1.5131 + }
1.5132 + // Verify that we have no work before we resort to stealing
1.5133 + assert(work_q->size() == 0, "Have work, shouldn't steal");
1.5134 + // Try to steal from other queues that have work
1.5135 + if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
1.5136 + NOT_PRODUCT(num_steals++;)
1.5137 + assert(obj_to_scan->is_oop(), "Oops, not an oop!");
1.5138 + assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
1.5139 + // Do scanning work
1.5140 + obj_to_scan->oop_iterate(cl);
1.5141 + // Loop around, finish this work, and try to steal some more
1.5142 + } else if (terminator()->offer_termination()) {
1.5143 + break; // nirvana from the infinite cycle
1.5144 + }
1.5145 + }
1.5146 + NOT_PRODUCT(
1.5147 + if (PrintCMSStatistics != 0) {
1.5148 + gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
1.5149 + }
1.5150 + )
1.5151 + assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
1.5152 + "Else our work is not yet done");
1.5153 +}
1.5154 +
1.5155 +// Return a thread-local PLAB recording array, as appropriate.
1.5156 +void* CMSCollector::get_data_recorder(int thr_num) {
1.5157 + if (_survivor_plab_array != NULL &&
1.5158 + (CMSPLABRecordAlways ||
1.5159 + (_collectorState > Marking && _collectorState < FinalMarking))) {
1.5160 + assert(thr_num < (int)ParallelGCThreads, "thr_num is out of bounds");
1.5161 + ChunkArray* ca = &_survivor_plab_array[thr_num];
1.5162 + ca->reset(); // clear it so that fresh data is recorded
1.5163 + return (void*) ca;
1.5164 + } else {
1.5165 + return NULL;
1.5166 + }
1.5167 +}
1.5168 +
1.5169 +// Reset all the thread-local PLAB recording arrays
1.5170 +void CMSCollector::reset_survivor_plab_arrays() {
1.5171 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.5172 + _survivor_plab_array[i].reset();
1.5173 + }
1.5174 +}
1.5175 +
1.5176 +// Merge the per-thread plab arrays into the global survivor chunk
1.5177 +// array which will provide the partitioning of the survivor space
1.5178 +// for CMS rescan.
1.5179 +void CMSCollector::merge_survivor_plab_arrays(ContiguousSpace* surv) {
1.5180 + assert(_survivor_plab_array != NULL, "Error");
1.5181 + assert(_survivor_chunk_array != NULL, "Error");
1.5182 + assert(_collectorState == FinalMarking, "Error");
1.5183 + for (uint j = 0; j < ParallelGCThreads; j++) {
1.5184 + _cursor[j] = 0;
1.5185 + }
1.5186 + HeapWord* top = surv->top();
1.5187 + size_t i;
1.5188 + for (i = 0; i < _survivor_chunk_capacity; i++) { // all sca entries
1.5189 + HeapWord* min_val = top; // Higher than any PLAB address
1.5190 + uint min_tid = 0; // position of min_val this round
1.5191 + for (uint j = 0; j < ParallelGCThreads; j++) {
1.5192 + ChunkArray* cur_sca = &_survivor_plab_array[j];
1.5193 + if (_cursor[j] == cur_sca->end()) {
1.5194 + continue;
1.5195 + }
1.5196 + assert(_cursor[j] < cur_sca->end(), "ctl pt invariant");
1.5197 + HeapWord* cur_val = cur_sca->nth(_cursor[j]);
1.5198 + assert(surv->used_region().contains(cur_val), "Out of bounds value");
1.5199 + if (cur_val < min_val) {
1.5200 + min_tid = j;
1.5201 + min_val = cur_val;
1.5202 + } else {
1.5203 + assert(cur_val < top, "All recorded addresses should be less");
1.5204 + }
1.5205 + }
1.5206 + // At this point min_val and min_tid are respectively
1.5207 + // the least address in _survivor_plab_array[j]->nth(_cursor[j])
1.5208 + // and the thread (j) that witnesses that address.
1.5209 + // We record this address in the _survivor_chunk_array[i]
1.5210 + // and increment _cursor[min_tid] prior to the next round i.
1.5211 + if (min_val == top) {
1.5212 + break;
1.5213 + }
1.5214 + _survivor_chunk_array[i] = min_val;
1.5215 + _cursor[min_tid]++;
1.5216 + }
1.5217 + // We are all done; record the size of the _survivor_chunk_array
1.5218 + _survivor_chunk_index = i; // exclusive: [0, i)
1.5219 + if (PrintCMSStatistics > 0) {
1.5220 + gclog_or_tty->print(" (Survivor:" SIZE_FORMAT "chunks) ", i);
1.5221 + }
1.5222 + // Verify that we used up all the recorded entries
1.5223 + #ifdef ASSERT
1.5224 + size_t total = 0;
1.5225 + for (uint j = 0; j < ParallelGCThreads; j++) {
1.5226 + assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
1.5227 + total += _cursor[j];
1.5228 + }
1.5229 + assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
1.5230 + // Check that the merged array is in sorted order
1.5231 + if (total > 0) {
1.5232 + for (size_t i = 0; i < total - 1; i++) {
1.5233 + if (PrintCMSStatistics > 0) {
1.5234 + gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
1.5235 + i, _survivor_chunk_array[i]);
1.5236 + }
1.5237 + assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
1.5238 + "Not sorted");
1.5239 + }
1.5240 + }
1.5241 + #endif // ASSERT
1.5242 +}
1.5243 +
1.5244 +// Set up the space's par_seq_tasks structure for work claiming
1.5245 +// for parallel rescan of young gen.
1.5246 +// See ParRescanTask where this is currently used.
1.5247 +void
1.5248 +CMSCollector::
1.5249 +initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
1.5250 + assert(n_threads > 0, "Unexpected n_threads argument");
1.5251 + DefNewGeneration* dng = (DefNewGeneration*)_young_gen;
1.5252 +
1.5253 + // Eden space
1.5254 + {
1.5255 + SequentialSubTasksDone* pst = dng->eden()->par_seq_tasks();
1.5256 + assert(!pst->valid(), "Clobbering existing data?");
1.5257 + // Each valid entry in [0, _eden_chunk_index) represents a task.
1.5258 + size_t n_tasks = _eden_chunk_index + 1;
1.5259 + assert(n_tasks == 1 || _eden_chunk_array != NULL, "Error");
1.5260 + pst->set_par_threads(n_threads);
1.5261 + pst->set_n_tasks((int)n_tasks);
1.5262 + }
1.5263 +
1.5264 + // Merge the survivor plab arrays into _survivor_chunk_array
1.5265 + if (_survivor_plab_array != NULL) {
1.5266 + merge_survivor_plab_arrays(dng->from());
1.5267 + } else {
1.5268 + assert(_survivor_chunk_index == 0, "Error");
1.5269 + }
1.5270 +
1.5271 + // To space
1.5272 + {
1.5273 + SequentialSubTasksDone* pst = dng->to()->par_seq_tasks();
1.5274 + assert(!pst->valid(), "Clobbering existing data?");
1.5275 + pst->set_par_threads(n_threads);
1.5276 + pst->set_n_tasks(1);
1.5277 + assert(pst->valid(), "Error");
1.5278 + }
1.5279 +
1.5280 + // From space
1.5281 + {
1.5282 + SequentialSubTasksDone* pst = dng->from()->par_seq_tasks();
1.5283 + assert(!pst->valid(), "Clobbering existing data?");
1.5284 + size_t n_tasks = _survivor_chunk_index + 1;
1.5285 + assert(n_tasks == 1 || _survivor_chunk_array != NULL, "Error");
1.5286 + pst->set_par_threads(n_threads);
1.5287 + pst->set_n_tasks((int)n_tasks);
1.5288 + assert(pst->valid(), "Error");
1.5289 + }
1.5290 +}
1.5291 +
1.5292 +// Parallel version of remark
1.5293 +void CMSCollector::do_remark_parallel() {
1.5294 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5295 + WorkGang* workers = gch->workers();
1.5296 + assert(workers != NULL, "Need parallel worker threads.");
1.5297 + int n_workers = workers->total_workers();
1.5298 + CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
1.5299 + CompactibleFreeListSpace* perm_space = _permGen->cmsSpace();
1.5300 +
1.5301 + CMSParRemarkTask tsk(this,
1.5302 + cms_space, perm_space,
1.5303 + n_workers, workers, task_queues());
1.5304 +
1.5305 + // Set up for parallel process_strong_roots work.
1.5306 + gch->set_par_threads(n_workers);
1.5307 + gch->change_strong_roots_parity();
1.5308 + // We won't be iterating over the cards in the card table updating
1.5309 + // the younger_gen cards, so we shouldn't call the following else
1.5310 + // the verification code as well as subsequent younger_refs_iterate
1.5311 + // code would get confused. XXX
1.5312 + // gch->rem_set()->prepare_for_younger_refs_iterate(true); // parallel
1.5313 +
1.5314 + // The young gen rescan work will not be done as part of
1.5315 + // process_strong_roots (which currently doesn't knw how to
1.5316 + // parallelize such a scan), but rather will be broken up into
1.5317 + // a set of parallel tasks (via the sampling that the [abortable]
1.5318 + // preclean phase did of EdenSpace, plus the [two] tasks of
1.5319 + // scanning the [two] survivor spaces. Further fine-grain
1.5320 + // parallelization of the scanning of the survivor spaces
1.5321 + // themselves, and of precleaning of the younger gen itself
1.5322 + // is deferred to the future.
1.5323 + initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
1.5324 +
1.5325 + // The dirty card rescan work is broken up into a "sequence"
1.5326 + // of parallel tasks (per constituent space) that are dynamically
1.5327 + // claimed by the parallel threads.
1.5328 + cms_space->initialize_sequential_subtasks_for_rescan(n_workers);
1.5329 + perm_space->initialize_sequential_subtasks_for_rescan(n_workers);
1.5330 +
1.5331 + // It turns out that even when we're using 1 thread, doing the work in a
1.5332 + // separate thread causes wide variance in run times. We can't help this
1.5333 + // in the multi-threaded case, but we special-case n=1 here to get
1.5334 + // repeatable measurements of the 1-thread overhead of the parallel code.
1.5335 + if (n_workers > 1) {
1.5336 + // Make refs discovery MT-safe
1.5337 + ReferenceProcessorMTMutator mt(ref_processor(), true);
1.5338 + workers->run_task(&tsk);
1.5339 + } else {
1.5340 + tsk.work(0);
1.5341 + }
1.5342 + gch->set_par_threads(0); // 0 ==> non-parallel.
1.5343 + // restore, single-threaded for now, any preserved marks
1.5344 + // as a result of work_q overflow
1.5345 + restore_preserved_marks_if_any();
1.5346 +}
1.5347 +
1.5348 +// Non-parallel version of remark
1.5349 +void CMSCollector::do_remark_non_parallel() {
1.5350 + ResourceMark rm;
1.5351 + HandleMark hm;
1.5352 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5353 + MarkRefsIntoAndScanClosure
1.5354 + mrias_cl(_span, ref_processor(), &_markBitMap, &_modUnionTable,
1.5355 + &_markStack, &_revisitStack, this,
1.5356 + false /* should_yield */, false /* not precleaning */);
1.5357 + MarkFromDirtyCardsClosure
1.5358 + markFromDirtyCardsClosure(this, _span,
1.5359 + NULL, // space is set further below
1.5360 + &_markBitMap, &_markStack, &_revisitStack,
1.5361 + &mrias_cl);
1.5362 + {
1.5363 + TraceTime t("grey object rescan", PrintGCDetails, false, gclog_or_tty);
1.5364 + // Iterate over the dirty cards, marking them precleaned, and
1.5365 + // setting the corresponding bits in the mod union table.
1.5366 + {
1.5367 + ModUnionClosure modUnionClosure(&_modUnionTable);
1.5368 + _ct->ct_bs()->dirty_card_iterate(
1.5369 + _cmsGen->used_region(),
1.5370 + &modUnionClosure);
1.5371 + _ct->ct_bs()->dirty_card_iterate(
1.5372 + _permGen->used_region(),
1.5373 + &modUnionClosure);
1.5374 + }
1.5375 + // Having transferred these marks into the modUnionTable, we just need
1.5376 + // to rescan the marked objects on the dirty cards in the modUnionTable.
1.5377 + // The initial marking may have been done during an asynchronous
1.5378 + // collection so there may be dirty bits in the mod-union table.
1.5379 + const int alignment =
1.5380 + CardTableModRefBS::card_size * BitsPerWord;
1.5381 + {
1.5382 + // ... First handle dirty cards in CMS gen
1.5383 + markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
1.5384 + MemRegion ur = _cmsGen->used_region();
1.5385 + HeapWord* lb = ur.start();
1.5386 + HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
1.5387 + MemRegion cms_span(lb, ub);
1.5388 + _modUnionTable.dirty_range_iterate_clear(cms_span,
1.5389 + &markFromDirtyCardsClosure);
1.5390 + verify_work_stacks_empty();
1.5391 + if (PrintCMSStatistics != 0) {
1.5392 + gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ",
1.5393 + markFromDirtyCardsClosure.num_dirty_cards());
1.5394 + }
1.5395 + }
1.5396 + {
1.5397 + // .. and then repeat for dirty cards in perm gen
1.5398 + markFromDirtyCardsClosure.set_space(_permGen->cmsSpace());
1.5399 + MemRegion ur = _permGen->used_region();
1.5400 + HeapWord* lb = ur.start();
1.5401 + HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
1.5402 + MemRegion perm_span(lb, ub);
1.5403 + _modUnionTable.dirty_range_iterate_clear(perm_span,
1.5404 + &markFromDirtyCardsClosure);
1.5405 + verify_work_stacks_empty();
1.5406 + if (PrintCMSStatistics != 0) {
1.5407 + gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in perm gen) ",
1.5408 + markFromDirtyCardsClosure.num_dirty_cards());
1.5409 + }
1.5410 + }
1.5411 + }
1.5412 + if (VerifyDuringGC &&
1.5413 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.5414 + HandleMark hm; // Discard invalid handles created during verification
1.5415 + Universe::verify(true);
1.5416 + }
1.5417 + {
1.5418 + TraceTime t("root rescan", PrintGCDetails, false, gclog_or_tty);
1.5419 +
1.5420 + verify_work_stacks_empty();
1.5421 +
1.5422 + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
1.5423 + gch->gen_process_strong_roots(_cmsGen->level(),
1.5424 + true, // younger gens as roots
1.5425 + true, // collecting perm gen
1.5426 + SharedHeap::ScanningOption(roots_scanning_options()),
1.5427 + NULL, &mrias_cl);
1.5428 + }
1.5429 + verify_work_stacks_empty();
1.5430 + // Restore evacuated mark words, if any, used for overflow list links
1.5431 + if (!CMSOverflowEarlyRestoration) {
1.5432 + restore_preserved_marks_if_any();
1.5433 + }
1.5434 + verify_overflow_empty();
1.5435 +}
1.5436 +
1.5437 +////////////////////////////////////////////////////////
1.5438 +// Parallel Reference Processing Task Proxy Class
1.5439 +////////////////////////////////////////////////////////
1.5440 +class CMSRefProcTaskProxy: public AbstractGangTask {
1.5441 + typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
1.5442 + CMSCollector* _collector;
1.5443 + CMSBitMap* _mark_bit_map;
1.5444 + MemRegion _span;
1.5445 + OopTaskQueueSet* _task_queues;
1.5446 + ParallelTaskTerminator _term;
1.5447 + ProcessTask& _task;
1.5448 +
1.5449 +public:
1.5450 + CMSRefProcTaskProxy(ProcessTask& task,
1.5451 + CMSCollector* collector,
1.5452 + const MemRegion& span,
1.5453 + CMSBitMap* mark_bit_map,
1.5454 + int total_workers,
1.5455 + OopTaskQueueSet* task_queues):
1.5456 + AbstractGangTask("Process referents by policy in parallel"),
1.5457 + _task(task),
1.5458 + _collector(collector), _span(span), _mark_bit_map(mark_bit_map),
1.5459 + _task_queues(task_queues),
1.5460 + _term(total_workers, task_queues)
1.5461 + { }
1.5462 +
1.5463 + OopTaskQueueSet* task_queues() { return _task_queues; }
1.5464 +
1.5465 + OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
1.5466 +
1.5467 + ParallelTaskTerminator* terminator() { return &_term; }
1.5468 +
1.5469 + void do_work_steal(int i,
1.5470 + CMSParDrainMarkingStackClosure* drain,
1.5471 + CMSParKeepAliveClosure* keep_alive,
1.5472 + int* seed);
1.5473 +
1.5474 + virtual void work(int i);
1.5475 +};
1.5476 +
1.5477 +void CMSRefProcTaskProxy::work(int i) {
1.5478 + CMSParKeepAliveClosure par_keep_alive(_collector, _span,
1.5479 + _mark_bit_map, work_queue(i));
1.5480 + CMSParDrainMarkingStackClosure par_drain_stack(_collector, _span,
1.5481 + _mark_bit_map, work_queue(i));
1.5482 + CMSIsAliveClosure is_alive_closure(_mark_bit_map);
1.5483 + _task.work(i, is_alive_closure, par_keep_alive, par_drain_stack);
1.5484 + if (_task.marks_oops_alive()) {
1.5485 + do_work_steal(i, &par_drain_stack, &par_keep_alive,
1.5486 + _collector->hash_seed(i));
1.5487 + }
1.5488 + assert(work_queue(i)->size() == 0, "work_queue should be empty");
1.5489 + assert(_collector->_overflow_list == NULL, "non-empty _overflow_list");
1.5490 +}
1.5491 +
1.5492 +class CMSRefEnqueueTaskProxy: public AbstractGangTask {
1.5493 + typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
1.5494 + EnqueueTask& _task;
1.5495 +
1.5496 +public:
1.5497 + CMSRefEnqueueTaskProxy(EnqueueTask& task)
1.5498 + : AbstractGangTask("Enqueue reference objects in parallel"),
1.5499 + _task(task)
1.5500 + { }
1.5501 +
1.5502 + virtual void work(int i)
1.5503 + {
1.5504 + _task.work(i);
1.5505 + }
1.5506 +};
1.5507 +
1.5508 +CMSParKeepAliveClosure::CMSParKeepAliveClosure(CMSCollector* collector,
1.5509 + MemRegion span, CMSBitMap* bit_map, OopTaskQueue* work_queue):
1.5510 + _collector(collector),
1.5511 + _span(span),
1.5512 + _bit_map(bit_map),
1.5513 + _work_queue(work_queue),
1.5514 + _mark_and_push(collector, span, bit_map, work_queue),
1.5515 + _low_water_mark(MIN2((uint)(work_queue->max_elems()/4),
1.5516 + (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads)))
1.5517 +{ }
1.5518 +
1.5519 +// . see if we can share work_queues with ParNew? XXX
1.5520 +void CMSRefProcTaskProxy::do_work_steal(int i,
1.5521 + CMSParDrainMarkingStackClosure* drain,
1.5522 + CMSParKeepAliveClosure* keep_alive,
1.5523 + int* seed) {
1.5524 + OopTaskQueue* work_q = work_queue(i);
1.5525 + NOT_PRODUCT(int num_steals = 0;)
1.5526 + oop obj_to_scan;
1.5527 + size_t num_from_overflow_list =
1.5528 + MIN2((size_t)work_q->max_elems()/4,
1.5529 + (size_t)ParGCDesiredObjsFromOverflowList);
1.5530 +
1.5531 + while (true) {
1.5532 + // Completely finish any left over work from (an) earlier round(s)
1.5533 + drain->trim_queue(0);
1.5534 + // Now check if there's any work in the overflow list
1.5535 + if (_collector->par_take_from_overflow_list(num_from_overflow_list,
1.5536 + work_q)) {
1.5537 + // Found something in global overflow list;
1.5538 + // not yet ready to go stealing work from others.
1.5539 + // We'd like to assert(work_q->size() != 0, ...)
1.5540 + // because we just took work from the overflow list,
1.5541 + // but of course we can't, since all of that might have
1.5542 + // been already stolen from us.
1.5543 + continue;
1.5544 + }
1.5545 + // Verify that we have no work before we resort to stealing
1.5546 + assert(work_q->size() == 0, "Have work, shouldn't steal");
1.5547 + // Try to steal from other queues that have work
1.5548 + if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
1.5549 + NOT_PRODUCT(num_steals++;)
1.5550 + assert(obj_to_scan->is_oop(), "Oops, not an oop!");
1.5551 + assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
1.5552 + // Do scanning work
1.5553 + obj_to_scan->oop_iterate(keep_alive);
1.5554 + // Loop around, finish this work, and try to steal some more
1.5555 + } else if (terminator()->offer_termination()) {
1.5556 + break; // nirvana from the infinite cycle
1.5557 + }
1.5558 + }
1.5559 + NOT_PRODUCT(
1.5560 + if (PrintCMSStatistics != 0) {
1.5561 + gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
1.5562 + }
1.5563 + )
1.5564 +}
1.5565 +
1.5566 +void CMSRefProcTaskExecutor::execute(ProcessTask& task)
1.5567 +{
1.5568 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5569 + WorkGang* workers = gch->workers();
1.5570 + assert(workers != NULL, "Need parallel worker threads.");
1.5571 + int n_workers = workers->total_workers();
1.5572 + CMSRefProcTaskProxy rp_task(task, &_collector,
1.5573 + _collector.ref_processor()->span(),
1.5574 + _collector.markBitMap(),
1.5575 + n_workers, _collector.task_queues());
1.5576 + workers->run_task(&rp_task);
1.5577 +}
1.5578 +
1.5579 +void CMSRefProcTaskExecutor::execute(EnqueueTask& task)
1.5580 +{
1.5581 +
1.5582 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5583 + WorkGang* workers = gch->workers();
1.5584 + assert(workers != NULL, "Need parallel worker threads.");
1.5585 + CMSRefEnqueueTaskProxy enq_task(task);
1.5586 + workers->run_task(&enq_task);
1.5587 +}
1.5588 +
1.5589 +void CMSCollector::refProcessingWork(bool asynch, bool clear_all_soft_refs) {
1.5590 +
1.5591 + ResourceMark rm;
1.5592 + HandleMark hm;
1.5593 + ReferencePolicy* soft_ref_policy;
1.5594 +
1.5595 + assert(!ref_processor()->enqueuing_is_done(), "Enqueuing should not be complete");
1.5596 + // Process weak references.
1.5597 + if (clear_all_soft_refs) {
1.5598 + soft_ref_policy = new AlwaysClearPolicy();
1.5599 + } else {
1.5600 +#ifdef COMPILER2
1.5601 + soft_ref_policy = new LRUMaxHeapPolicy();
1.5602 +#else
1.5603 + soft_ref_policy = new LRUCurrentHeapPolicy();
1.5604 +#endif // COMPILER2
1.5605 + }
1.5606 + verify_work_stacks_empty();
1.5607 +
1.5608 + ReferenceProcessor* rp = ref_processor();
1.5609 + assert(rp->span().equals(_span), "Spans should be equal");
1.5610 + CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap,
1.5611 + &_markStack);
1.5612 + CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this,
1.5613 + _span, &_markBitMap, &_markStack,
1.5614 + &cmsKeepAliveClosure);
1.5615 + {
1.5616 + TraceTime t("weak refs processing", PrintGCDetails, false, gclog_or_tty);
1.5617 + if (rp->processing_is_mt()) {
1.5618 + CMSRefProcTaskExecutor task_executor(*this);
1.5619 + rp->process_discovered_references(soft_ref_policy,
1.5620 + &_is_alive_closure,
1.5621 + &cmsKeepAliveClosure,
1.5622 + &cmsDrainMarkingStackClosure,
1.5623 + &task_executor);
1.5624 + } else {
1.5625 + rp->process_discovered_references(soft_ref_policy,
1.5626 + &_is_alive_closure,
1.5627 + &cmsKeepAliveClosure,
1.5628 + &cmsDrainMarkingStackClosure,
1.5629 + NULL);
1.5630 + }
1.5631 + verify_work_stacks_empty();
1.5632 + }
1.5633 +
1.5634 + if (cms_should_unload_classes()) {
1.5635 + {
1.5636 + TraceTime t("class unloading", PrintGCDetails, false, gclog_or_tty);
1.5637 +
1.5638 + // Follow SystemDictionary roots and unload classes
1.5639 + bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure);
1.5640 +
1.5641 + // Follow CodeCache roots and unload any methods marked for unloading
1.5642 + CodeCache::do_unloading(&_is_alive_closure,
1.5643 + &cmsKeepAliveClosure,
1.5644 + purged_class);
1.5645 +
1.5646 + cmsDrainMarkingStackClosure.do_void();
1.5647 + verify_work_stacks_empty();
1.5648 +
1.5649 + // Update subklass/sibling/implementor links in KlassKlass descendants
1.5650 + assert(!_revisitStack.isEmpty(), "revisit stack should not be empty");
1.5651 + oop k;
1.5652 + while ((k = _revisitStack.pop()) != NULL) {
1.5653 + ((Klass*)(oopDesc*)k)->follow_weak_klass_links(
1.5654 + &_is_alive_closure,
1.5655 + &cmsKeepAliveClosure);
1.5656 + }
1.5657 + assert(!ClassUnloading ||
1.5658 + (_markStack.isEmpty() && overflow_list_is_empty()),
1.5659 + "Should not have found new reachable objects");
1.5660 + assert(_revisitStack.isEmpty(), "revisit stack should have been drained");
1.5661 + cmsDrainMarkingStackClosure.do_void();
1.5662 + verify_work_stacks_empty();
1.5663 + }
1.5664 +
1.5665 + {
1.5666 + TraceTime t("scrub symbol & string tables", PrintGCDetails, false, gclog_or_tty);
1.5667 + // Now clean up stale oops in SymbolTable and StringTable
1.5668 + SymbolTable::unlink(&_is_alive_closure);
1.5669 + StringTable::unlink(&_is_alive_closure);
1.5670 + }
1.5671 + }
1.5672 +
1.5673 + verify_work_stacks_empty();
1.5674 + // Restore any preserved marks as a result of mark stack or
1.5675 + // work queue overflow
1.5676 + restore_preserved_marks_if_any(); // done single-threaded for now
1.5677 +
1.5678 + rp->set_enqueuing_is_done(true);
1.5679 + if (rp->processing_is_mt()) {
1.5680 + CMSRefProcTaskExecutor task_executor(*this);
1.5681 + rp->enqueue_discovered_references(&task_executor);
1.5682 + } else {
1.5683 + rp->enqueue_discovered_references(NULL);
1.5684 + }
1.5685 + rp->verify_no_references_recorded();
1.5686 + assert(!rp->discovery_enabled(), "should have been disabled");
1.5687 +
1.5688 + // JVMTI object tagging is based on JNI weak refs. If any of these
1.5689 + // refs were cleared then JVMTI needs to update its maps and
1.5690 + // maybe post ObjectFrees to agents.
1.5691 + JvmtiExport::cms_ref_processing_epilogue();
1.5692 +}
1.5693 +
1.5694 +#ifndef PRODUCT
1.5695 +void CMSCollector::check_correct_thread_executing() {
1.5696 + Thread* t = Thread::current();
1.5697 + // Only the VM thread or the CMS thread should be here.
1.5698 + assert(t->is_ConcurrentGC_thread() || t->is_VM_thread(),
1.5699 + "Unexpected thread type");
1.5700 + // If this is the vm thread, the foreground process
1.5701 + // should not be waiting. Note that _foregroundGCIsActive is
1.5702 + // true while the foreground collector is waiting.
1.5703 + if (_foregroundGCShouldWait) {
1.5704 + // We cannot be the VM thread
1.5705 + assert(t->is_ConcurrentGC_thread(),
1.5706 + "Should be CMS thread");
1.5707 + } else {
1.5708 + // We can be the CMS thread only if we are in a stop-world
1.5709 + // phase of CMS collection.
1.5710 + if (t->is_ConcurrentGC_thread()) {
1.5711 + assert(_collectorState == InitialMarking ||
1.5712 + _collectorState == FinalMarking,
1.5713 + "Should be a stop-world phase");
1.5714 + // The CMS thread should be holding the CMS_token.
1.5715 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.5716 + "Potential interference with concurrently "
1.5717 + "executing VM thread");
1.5718 + }
1.5719 + }
1.5720 +}
1.5721 +#endif
1.5722 +
1.5723 +void CMSCollector::sweep(bool asynch) {
1.5724 + assert(_collectorState == Sweeping, "just checking");
1.5725 + check_correct_thread_executing();
1.5726 + verify_work_stacks_empty();
1.5727 + verify_overflow_empty();
1.5728 + incrementSweepCount();
1.5729 + _sweep_timer.stop();
1.5730 + _sweep_estimate.sample(_sweep_timer.seconds());
1.5731 + size_policy()->avg_cms_free_at_sweep()->sample(_cmsGen->free());
1.5732 +
1.5733 + // PermGen verification support: If perm gen sweeping is disabled in
1.5734 + // this cycle, we preserve the perm gen object "deadness" information
1.5735 + // in the perm_gen_verify_bit_map. In order to do that we traverse
1.5736 + // all blocks in perm gen and mark all dead objects.
1.5737 + if (verifying() && !cms_should_unload_classes()) {
1.5738 + CMSTokenSyncWithLocks ts(true, _permGen->freelistLock(),
1.5739 + bitMapLock());
1.5740 + assert(perm_gen_verify_bit_map()->sizeInBits() != 0,
1.5741 + "Should have already been allocated");
1.5742 + MarkDeadObjectsClosure mdo(this, _permGen->cmsSpace(),
1.5743 + markBitMap(), perm_gen_verify_bit_map());
1.5744 + _permGen->cmsSpace()->blk_iterate(&mdo);
1.5745 + }
1.5746 +
1.5747 + if (asynch) {
1.5748 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.5749 + CMSPhaseAccounting pa(this, "sweep", !PrintGCDetails);
1.5750 + // First sweep the old gen then the perm gen
1.5751 + {
1.5752 + CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
1.5753 + bitMapLock());
1.5754 + sweepWork(_cmsGen, asynch);
1.5755 + }
1.5756 +
1.5757 + // Now repeat for perm gen
1.5758 + if (cms_should_unload_classes()) {
1.5759 + CMSTokenSyncWithLocks ts(true, _permGen->freelistLock(),
1.5760 + bitMapLock());
1.5761 + sweepWork(_permGen, asynch);
1.5762 + }
1.5763 +
1.5764 + // Update Universe::_heap_*_at_gc figures.
1.5765 + // We need all the free list locks to make the abstract state
1.5766 + // transition from Sweeping to Resetting. See detailed note
1.5767 + // further below.
1.5768 + {
1.5769 + CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
1.5770 + _permGen->freelistLock());
1.5771 + // Update heap occupancy information which is used as
1.5772 + // input to soft ref clearing policy at the next gc.
1.5773 + Universe::update_heap_info_at_gc();
1.5774 + _collectorState = Resizing;
1.5775 + }
1.5776 + } else {
1.5777 + // already have needed locks
1.5778 + sweepWork(_cmsGen, asynch);
1.5779 +
1.5780 + if (cms_should_unload_classes()) {
1.5781 + sweepWork(_permGen, asynch);
1.5782 + }
1.5783 + // Update heap occupancy information which is used as
1.5784 + // input to soft ref clearing policy at the next gc.
1.5785 + Universe::update_heap_info_at_gc();
1.5786 + _collectorState = Resizing;
1.5787 + }
1.5788 + verify_work_stacks_empty();
1.5789 + verify_overflow_empty();
1.5790 +
1.5791 + _sweep_timer.reset();
1.5792 + _sweep_timer.start();
1.5793 +
1.5794 + update_time_of_last_gc(os::javaTimeMillis());
1.5795 +
1.5796 + // NOTE on abstract state transitions:
1.5797 + // Mutators allocate-live and/or mark the mod-union table dirty
1.5798 + // based on the state of the collection. The former is done in
1.5799 + // the interval [Marking, Sweeping] and the latter in the interval
1.5800 + // [Marking, Sweeping). Thus the transitions into the Marking state
1.5801 + // and out of the Sweeping state must be synchronously visible
1.5802 + // globally to the mutators.
1.5803 + // The transition into the Marking state happens with the world
1.5804 + // stopped so the mutators will globally see it. Sweeping is
1.5805 + // done asynchronously by the background collector so the transition
1.5806 + // from the Sweeping state to the Resizing state must be done
1.5807 + // under the freelistLock (as is the check for whether to
1.5808 + // allocate-live and whether to dirty the mod-union table).
1.5809 + assert(_collectorState == Resizing, "Change of collector state to"
1.5810 + " Resizing must be done under the freelistLocks (plural)");
1.5811 +
1.5812 + // Now that sweeping has been completed, if the GCH's
1.5813 + // incremental_collection_will_fail flag is set, clear it,
1.5814 + // thus inviting a younger gen collection to promote into
1.5815 + // this generation. If such a promotion may still fail,
1.5816 + // the flag will be set again when a young collection is
1.5817 + // attempted.
1.5818 + // I think the incremental_collection_will_fail flag's use
1.5819 + // is specific to a 2 generation collection policy, so i'll
1.5820 + // assert that that's the configuration we are operating within.
1.5821 + // The use of the flag can and should be generalized appropriately
1.5822 + // in the future to deal with a general n-generation system.
1.5823 +
1.5824 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5825 + assert(gch->collector_policy()->is_two_generation_policy(),
1.5826 + "Resetting of incremental_collection_will_fail flag"
1.5827 + " may be incorrect otherwise");
1.5828 + gch->clear_incremental_collection_will_fail();
1.5829 + gch->update_full_collections_completed(_collection_count_start);
1.5830 +}
1.5831 +
1.5832 +// FIX ME!!! Looks like this belongs in CFLSpace, with
1.5833 +// CMSGen merely delegating to it.
1.5834 +void ConcurrentMarkSweepGeneration::setNearLargestChunk() {
1.5835 + double nearLargestPercent = 0.999;
1.5836 + HeapWord* minAddr = _cmsSpace->bottom();
1.5837 + HeapWord* largestAddr =
1.5838 + (HeapWord*) _cmsSpace->dictionary()->findLargestDict();
1.5839 + if (largestAddr == 0) {
1.5840 + // The dictionary appears to be empty. In this case
1.5841 + // try to coalesce at the end of the heap.
1.5842 + largestAddr = _cmsSpace->end();
1.5843 + }
1.5844 + size_t largestOffset = pointer_delta(largestAddr, minAddr);
1.5845 + size_t nearLargestOffset =
1.5846 + (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
1.5847 + _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
1.5848 +}
1.5849 +
1.5850 +bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
1.5851 + return addr >= _cmsSpace->nearLargestChunk();
1.5852 +}
1.5853 +
1.5854 +FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
1.5855 + return _cmsSpace->find_chunk_at_end();
1.5856 +}
1.5857 +
1.5858 +void ConcurrentMarkSweepGeneration::update_gc_stats(int current_level,
1.5859 + bool full) {
1.5860 + // The next lower level has been collected. Gather any statistics
1.5861 + // that are of interest at this point.
1.5862 + if (!full && (current_level + 1) == level()) {
1.5863 + // Gather statistics on the young generation collection.
1.5864 + collector()->stats().record_gc0_end(used());
1.5865 + }
1.5866 +}
1.5867 +
1.5868 +CMSAdaptiveSizePolicy* ConcurrentMarkSweepGeneration::size_policy() {
1.5869 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5870 + assert(gch->kind() == CollectedHeap::GenCollectedHeap,
1.5871 + "Wrong type of heap");
1.5872 + CMSAdaptiveSizePolicy* sp = (CMSAdaptiveSizePolicy*)
1.5873 + gch->gen_policy()->size_policy();
1.5874 + assert(sp->is_gc_cms_adaptive_size_policy(),
1.5875 + "Wrong type of size policy");
1.5876 + return sp;
1.5877 +}
1.5878 +
1.5879 +void ConcurrentMarkSweepGeneration::rotate_debug_collection_type() {
1.5880 + if (PrintGCDetails && Verbose) {
1.5881 + gclog_or_tty->print("Rotate from %d ", _debug_collection_type);
1.5882 + }
1.5883 + _debug_collection_type = (CollectionTypes) (_debug_collection_type + 1);
1.5884 + _debug_collection_type =
1.5885 + (CollectionTypes) (_debug_collection_type % Unknown_collection_type);
1.5886 + if (PrintGCDetails && Verbose) {
1.5887 + gclog_or_tty->print_cr("to %d ", _debug_collection_type);
1.5888 + }
1.5889 +}
1.5890 +
1.5891 +void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* gen,
1.5892 + bool asynch) {
1.5893 + // We iterate over the space(s) underlying this generation,
1.5894 + // checking the mark bit map to see if the bits corresponding
1.5895 + // to specific blocks are marked or not. Blocks that are
1.5896 + // marked are live and are not swept up. All remaining blocks
1.5897 + // are swept up, with coalescing on-the-fly as we sweep up
1.5898 + // contiguous free and/or garbage blocks:
1.5899 + // We need to ensure that the sweeper synchronizes with allocators
1.5900 + // and stop-the-world collectors. In particular, the following
1.5901 + // locks are used:
1.5902 + // . CMS token: if this is held, a stop the world collection cannot occur
1.5903 + // . freelistLock: if this is held no allocation can occur from this
1.5904 + // generation by another thread
1.5905 + // . bitMapLock: if this is held, no other thread can access or update
1.5906 + //
1.5907 +
1.5908 + // Note that we need to hold the freelistLock if we use
1.5909 + // block iterate below; else the iterator might go awry if
1.5910 + // a mutator (or promotion) causes block contents to change
1.5911 + // (for instance if the allocator divvies up a block).
1.5912 + // If we hold the free list lock, for all practical purposes
1.5913 + // young generation GC's can't occur (they'll usually need to
1.5914 + // promote), so we might as well prevent all young generation
1.5915 + // GC's while we do a sweeping step. For the same reason, we might
1.5916 + // as well take the bit map lock for the entire duration
1.5917 +
1.5918 + // check that we hold the requisite locks
1.5919 + assert(have_cms_token(), "Should hold cms token");
1.5920 + assert( (asynch && ConcurrentMarkSweepThread::cms_thread_has_cms_token())
1.5921 + || (!asynch && ConcurrentMarkSweepThread::vm_thread_has_cms_token()),
1.5922 + "Should possess CMS token to sweep");
1.5923 + assert_lock_strong(gen->freelistLock());
1.5924 + assert_lock_strong(bitMapLock());
1.5925 +
1.5926 + assert(!_sweep_timer.is_active(), "Was switched off in an outer context");
1.5927 + gen->cmsSpace()->beginSweepFLCensus((float)(_sweep_timer.seconds()),
1.5928 + _sweep_estimate.padded_average());
1.5929 + gen->setNearLargestChunk();
1.5930 +
1.5931 + {
1.5932 + SweepClosure sweepClosure(this, gen, &_markBitMap,
1.5933 + CMSYield && asynch);
1.5934 + gen->cmsSpace()->blk_iterate_careful(&sweepClosure);
1.5935 + // We need to free-up/coalesce garbage/blocks from a
1.5936 + // co-terminal free run. This is done in the SweepClosure
1.5937 + // destructor; so, do not remove this scope, else the
1.5938 + // end-of-sweep-census below will be off by a little bit.
1.5939 + }
1.5940 + gen->cmsSpace()->sweep_completed();
1.5941 + gen->cmsSpace()->endSweepFLCensus(sweepCount());
1.5942 +}
1.5943 +
1.5944 +// Reset CMS data structures (for now just the marking bit map)
1.5945 +// preparatory for the next cycle.
1.5946 +void CMSCollector::reset(bool asynch) {
1.5947 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5948 + CMSAdaptiveSizePolicy* sp = size_policy();
1.5949 + AdaptiveSizePolicyOutput(sp, gch->total_collections());
1.5950 + if (asynch) {
1.5951 + CMSTokenSyncWithLocks ts(true, bitMapLock());
1.5952 +
1.5953 + // If the state is not "Resetting", the foreground thread
1.5954 + // has done a collection and the resetting.
1.5955 + if (_collectorState != Resetting) {
1.5956 + assert(_collectorState == Idling, "The state should only change"
1.5957 + " because the foreground collector has finished the collection");
1.5958 + return;
1.5959 + }
1.5960 +
1.5961 + // Clear the mark bitmap (no grey objects to start with)
1.5962 + // for the next cycle.
1.5963 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.5964 + CMSPhaseAccounting cmspa(this, "reset", !PrintGCDetails);
1.5965 +
1.5966 + HeapWord* curAddr = _markBitMap.startWord();
1.5967 + while (curAddr < _markBitMap.endWord()) {
1.5968 + size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr);
1.5969 + MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining));
1.5970 + _markBitMap.clear_large_range(chunk);
1.5971 + if (ConcurrentMarkSweepThread::should_yield() &&
1.5972 + !foregroundGCIsActive() &&
1.5973 + CMSYield) {
1.5974 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.5975 + "CMS thread should hold CMS token");
1.5976 + assert_lock_strong(bitMapLock());
1.5977 + bitMapLock()->unlock();
1.5978 + ConcurrentMarkSweepThread::desynchronize(true);
1.5979 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.5980 + stopTimer();
1.5981 + if (PrintCMSStatistics != 0) {
1.5982 + incrementYields();
1.5983 + }
1.5984 + icms_wait();
1.5985 +
1.5986 + // See the comment in coordinator_yield()
1.5987 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.5988 + ConcurrentMarkSweepThread::should_yield() &&
1.5989 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.5990 + os::sleep(Thread::current(), 1, false);
1.5991 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.5992 + }
1.5993 +
1.5994 + ConcurrentMarkSweepThread::synchronize(true);
1.5995 + bitMapLock()->lock_without_safepoint_check();
1.5996 + startTimer();
1.5997 + }
1.5998 + curAddr = chunk.end();
1.5999 + }
1.6000 + _collectorState = Idling;
1.6001 + } else {
1.6002 + // already have the lock
1.6003 + assert(_collectorState == Resetting, "just checking");
1.6004 + assert_lock_strong(bitMapLock());
1.6005 + _markBitMap.clear_all();
1.6006 + _collectorState = Idling;
1.6007 + }
1.6008 +
1.6009 + // Stop incremental mode after a cycle completes, so that any future cycles
1.6010 + // are triggered by allocation.
1.6011 + stop_icms();
1.6012 +
1.6013 + NOT_PRODUCT(
1.6014 + if (RotateCMSCollectionTypes) {
1.6015 + _cmsGen->rotate_debug_collection_type();
1.6016 + }
1.6017 + )
1.6018 +}
1.6019 +
1.6020 +void CMSCollector::do_CMS_operation(CMS_op_type op) {
1.6021 + gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
1.6022 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.6023 + TraceTime t("GC", PrintGC, !PrintGCDetails, gclog_or_tty);
1.6024 + TraceCollectorStats tcs(counters());
1.6025 +
1.6026 + switch (op) {
1.6027 + case CMS_op_checkpointRootsInitial: {
1.6028 + checkpointRootsInitial(true); // asynch
1.6029 + if (PrintGC) {
1.6030 + _cmsGen->printOccupancy("initial-mark");
1.6031 + }
1.6032 + break;
1.6033 + }
1.6034 + case CMS_op_checkpointRootsFinal: {
1.6035 + checkpointRootsFinal(true, // asynch
1.6036 + false, // !clear_all_soft_refs
1.6037 + false); // !init_mark_was_synchronous
1.6038 + if (PrintGC) {
1.6039 + _cmsGen->printOccupancy("remark");
1.6040 + }
1.6041 + break;
1.6042 + }
1.6043 + default:
1.6044 + fatal("No such CMS_op");
1.6045 + }
1.6046 +}
1.6047 +
1.6048 +#ifndef PRODUCT
1.6049 +size_t const CMSCollector::skip_header_HeapWords() {
1.6050 + return FreeChunk::header_size();
1.6051 +}
1.6052 +
1.6053 +// Try and collect here conditions that should hold when
1.6054 +// CMS thread is exiting. The idea is that the foreground GC
1.6055 +// thread should not be blocked if it wants to terminate
1.6056 +// the CMS thread and yet continue to run the VM for a while
1.6057 +// after that.
1.6058 +void CMSCollector::verify_ok_to_terminate() const {
1.6059 + assert(Thread::current()->is_ConcurrentGC_thread(),
1.6060 + "should be called by CMS thread");
1.6061 + assert(!_foregroundGCShouldWait, "should be false");
1.6062 + // We could check here that all the various low-level locks
1.6063 + // are not held by the CMS thread, but that is overkill; see
1.6064 + // also CMSThread::verify_ok_to_terminate() where the CGC_lock
1.6065 + // is checked.
1.6066 +}
1.6067 +#endif
1.6068 +
1.6069 +size_t CMSCollector::block_size_using_printezis_bits(HeapWord* addr) const {
1.6070 + assert(_markBitMap.isMarked(addr) && _markBitMap.isMarked(addr + 1),
1.6071 + "missing Printezis mark?");
1.6072 + HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
1.6073 + size_t size = pointer_delta(nextOneAddr + 1, addr);
1.6074 + assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
1.6075 + "alignment problem");
1.6076 + assert(size >= 3, "Necessary for Printezis marks to work");
1.6077 + return size;
1.6078 +}
1.6079 +
1.6080 +// A variant of the above (block_size_using_printezis_bits()) except
1.6081 +// that we return 0 if the P-bits are not yet set.
1.6082 +size_t CMSCollector::block_size_if_printezis_bits(HeapWord* addr) const {
1.6083 + if (_markBitMap.isMarked(addr)) {
1.6084 + assert(_markBitMap.isMarked(addr + 1), "Missing Printezis bit?");
1.6085 + HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
1.6086 + size_t size = pointer_delta(nextOneAddr + 1, addr);
1.6087 + assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
1.6088 + "alignment problem");
1.6089 + assert(size >= 3, "Necessary for Printezis marks to work");
1.6090 + return size;
1.6091 + } else {
1.6092 + assert(!_markBitMap.isMarked(addr + 1), "Bit map inconsistency?");
1.6093 + return 0;
1.6094 + }
1.6095 +}
1.6096 +
1.6097 +HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
1.6098 + size_t sz = 0;
1.6099 + oop p = (oop)addr;
1.6100 + if (p->klass() != NULL && p->is_parsable()) {
1.6101 + sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
1.6102 + } else {
1.6103 + sz = block_size_using_printezis_bits(addr);
1.6104 + }
1.6105 + assert(sz > 0, "size must be nonzero");
1.6106 + HeapWord* next_block = addr + sz;
1.6107 + HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block,
1.6108 + CardTableModRefBS::card_size);
1.6109 + assert(round_down((uintptr_t)addr, CardTableModRefBS::card_size) <
1.6110 + round_down((uintptr_t)next_card, CardTableModRefBS::card_size),
1.6111 + "must be different cards");
1.6112 + return next_card;
1.6113 +}
1.6114 +
1.6115 +
1.6116 +// CMS Bit Map Wrapper /////////////////////////////////////////
1.6117 +
1.6118 +// Construct a CMS bit map infrastructure, but don't create the
1.6119 +// bit vector itself. That is done by a separate call CMSBitMap::allocate()
1.6120 +// further below.
1.6121 +CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
1.6122 + _bm(NULL,0),
1.6123 + _shifter(shifter),
1.6124 + _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true) : NULL)
1.6125 +{
1.6126 + _bmStartWord = 0;
1.6127 + _bmWordSize = 0;
1.6128 +}
1.6129 +
1.6130 +bool CMSBitMap::allocate(MemRegion mr) {
1.6131 + _bmStartWord = mr.start();
1.6132 + _bmWordSize = mr.word_size();
1.6133 + ReservedSpace brs(ReservedSpace::allocation_align_size_up(
1.6134 + (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1));
1.6135 + if (!brs.is_reserved()) {
1.6136 + warning("CMS bit map allocation failure");
1.6137 + return false;
1.6138 + }
1.6139 + // For now we'll just commit all of the bit map up fromt.
1.6140 + // Later on we'll try to be more parsimonious with swap.
1.6141 + if (!_virtual_space.initialize(brs, brs.size())) {
1.6142 + warning("CMS bit map backing store failure");
1.6143 + return false;
1.6144 + }
1.6145 + assert(_virtual_space.committed_size() == brs.size(),
1.6146 + "didn't reserve backing store for all of CMS bit map?");
1.6147 + _bm.set_map((uintptr_t*)_virtual_space.low());
1.6148 + assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
1.6149 + _bmWordSize, "inconsistency in bit map sizing");
1.6150 + _bm.set_size(_bmWordSize >> _shifter);
1.6151 +
1.6152 + // bm.clear(); // can we rely on getting zero'd memory? verify below
1.6153 + assert(isAllClear(),
1.6154 + "Expected zero'd memory from ReservedSpace constructor");
1.6155 + assert(_bm.size() == heapWordDiffToOffsetDiff(sizeInWords()),
1.6156 + "consistency check");
1.6157 + return true;
1.6158 +}
1.6159 +
1.6160 +void CMSBitMap::dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl) {
1.6161 + HeapWord *next_addr, *end_addr, *last_addr;
1.6162 + assert_locked();
1.6163 + assert(covers(mr), "out-of-range error");
1.6164 + // XXX assert that start and end are appropriately aligned
1.6165 + for (next_addr = mr.start(), end_addr = mr.end();
1.6166 + next_addr < end_addr; next_addr = last_addr) {
1.6167 + MemRegion dirty_region = getAndClearMarkedRegion(next_addr, end_addr);
1.6168 + last_addr = dirty_region.end();
1.6169 + if (!dirty_region.is_empty()) {
1.6170 + cl->do_MemRegion(dirty_region);
1.6171 + } else {
1.6172 + assert(last_addr == end_addr, "program logic");
1.6173 + return;
1.6174 + }
1.6175 + }
1.6176 +}
1.6177 +
1.6178 +#ifndef PRODUCT
1.6179 +void CMSBitMap::assert_locked() const {
1.6180 + CMSLockVerifier::assert_locked(lock());
1.6181 +}
1.6182 +
1.6183 +bool CMSBitMap::covers(MemRegion mr) const {
1.6184 + // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
1.6185 + assert((size_t)_bm.size() == (_bmWordSize >> _shifter),
1.6186 + "size inconsistency");
1.6187 + return (mr.start() >= _bmStartWord) &&
1.6188 + (mr.end() <= endWord());
1.6189 +}
1.6190 +
1.6191 +bool CMSBitMap::covers(HeapWord* start, size_t size) const {
1.6192 + return (start >= _bmStartWord && (start + size) <= endWord());
1.6193 +}
1.6194 +
1.6195 +void CMSBitMap::verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) {
1.6196 + // verify that there are no 1 bits in the interval [left, right)
1.6197 + FalseBitMapClosure falseBitMapClosure;
1.6198 + iterate(&falseBitMapClosure, left, right);
1.6199 +}
1.6200 +
1.6201 +void CMSBitMap::region_invariant(MemRegion mr)
1.6202 +{
1.6203 + assert_locked();
1.6204 + // mr = mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
1.6205 + assert(!mr.is_empty(), "unexpected empty region");
1.6206 + assert(covers(mr), "mr should be covered by bit map");
1.6207 + // convert address range into offset range
1.6208 + size_t start_ofs = heapWordToOffset(mr.start());
1.6209 + // Make sure that end() is appropriately aligned
1.6210 + assert(mr.end() == (HeapWord*)round_to((intptr_t)mr.end(),
1.6211 + (1 << (_shifter+LogHeapWordSize))),
1.6212 + "Misaligned mr.end()");
1.6213 + size_t end_ofs = heapWordToOffset(mr.end());
1.6214 + assert(end_ofs > start_ofs, "Should mark at least one bit");
1.6215 +}
1.6216 +
1.6217 +#endif
1.6218 +
1.6219 +bool CMSMarkStack::allocate(size_t size) {
1.6220 + // allocate a stack of the requisite depth
1.6221 + ReservedSpace rs(ReservedSpace::allocation_align_size_up(
1.6222 + size * sizeof(oop)));
1.6223 + if (!rs.is_reserved()) {
1.6224 + warning("CMSMarkStack allocation failure");
1.6225 + return false;
1.6226 + }
1.6227 + if (!_virtual_space.initialize(rs, rs.size())) {
1.6228 + warning("CMSMarkStack backing store failure");
1.6229 + return false;
1.6230 + }
1.6231 + assert(_virtual_space.committed_size() == rs.size(),
1.6232 + "didn't reserve backing store for all of CMS stack?");
1.6233 + _base = (oop*)(_virtual_space.low());
1.6234 + _index = 0;
1.6235 + _capacity = size;
1.6236 + NOT_PRODUCT(_max_depth = 0);
1.6237 + return true;
1.6238 +}
1.6239 +
1.6240 +// XXX FIX ME !!! In the MT case we come in here holding a
1.6241 +// leaf lock. For printing we need to take a further lock
1.6242 +// which has lower rank. We need to recallibrate the two
1.6243 +// lock-ranks involved in order to be able to rpint the
1.6244 +// messages below. (Or defer the printing to the caller.
1.6245 +// For now we take the expedient path of just disabling the
1.6246 +// messages for the problematic case.)
1.6247 +void CMSMarkStack::expand() {
1.6248 + assert(_capacity <= CMSMarkStackSizeMax, "stack bigger than permitted");
1.6249 + if (_capacity == CMSMarkStackSizeMax) {
1.6250 + if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
1.6251 + // We print a warning message only once per CMS cycle.
1.6252 + gclog_or_tty->print_cr(" (benign) Hit CMSMarkStack max size limit");
1.6253 + }
1.6254 + return;
1.6255 + }
1.6256 + // Double capacity if possible
1.6257 + size_t new_capacity = MIN2(_capacity*2, CMSMarkStackSizeMax);
1.6258 + // Do not give up existing stack until we have managed to
1.6259 + // get the double capacity that we desired.
1.6260 + ReservedSpace rs(ReservedSpace::allocation_align_size_up(
1.6261 + new_capacity * sizeof(oop)));
1.6262 + if (rs.is_reserved()) {
1.6263 + // Release the backing store associated with old stack
1.6264 + _virtual_space.release();
1.6265 + // Reinitialize virtual space for new stack
1.6266 + if (!_virtual_space.initialize(rs, rs.size())) {
1.6267 + fatal("Not enough swap for expanded marking stack");
1.6268 + }
1.6269 + _base = (oop*)(_virtual_space.low());
1.6270 + _index = 0;
1.6271 + _capacity = new_capacity;
1.6272 + } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
1.6273 + // Failed to double capacity, continue;
1.6274 + // we print a detail message only once per CMS cycle.
1.6275 + gclog_or_tty->print(" (benign) Failed to expand marking stack from "SIZE_FORMAT"K to "
1.6276 + SIZE_FORMAT"K",
1.6277 + _capacity / K, new_capacity / K);
1.6278 + }
1.6279 +}
1.6280 +
1.6281 +
1.6282 +// Closures
1.6283 +// XXX: there seems to be a lot of code duplication here;
1.6284 +// should refactor and consolidate common code.
1.6285 +
1.6286 +// This closure is used to mark refs into the CMS generation in
1.6287 +// the CMS bit map. Called at the first checkpoint. This closure
1.6288 +// assumes that we do not need to re-mark dirty cards; if the CMS
1.6289 +// generation on which this is used is not an oldest (modulo perm gen)
1.6290 +// generation then this will lose younger_gen cards!
1.6291 +
1.6292 +MarkRefsIntoClosure::MarkRefsIntoClosure(
1.6293 + MemRegion span, CMSBitMap* bitMap, bool should_do_nmethods):
1.6294 + _span(span),
1.6295 + _bitMap(bitMap),
1.6296 + _should_do_nmethods(should_do_nmethods)
1.6297 +{
1.6298 + assert(_ref_processor == NULL, "deliberately left NULL");
1.6299 + assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
1.6300 +}
1.6301 +
1.6302 +void MarkRefsIntoClosure::do_oop(oop* p) {
1.6303 + // if p points into _span, then mark corresponding bit in _markBitMap
1.6304 + oop thisOop = *p;
1.6305 + if (thisOop != NULL) {
1.6306 + assert(thisOop->is_oop(), "expected an oop");
1.6307 + HeapWord* addr = (HeapWord*)thisOop;
1.6308 + if (_span.contains(addr)) {
1.6309 + // this should be made more efficient
1.6310 + _bitMap->mark(addr);
1.6311 + }
1.6312 + }
1.6313 +}
1.6314 +
1.6315 +// A variant of the above, used for CMS marking verification.
1.6316 +MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
1.6317 + MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm,
1.6318 + bool should_do_nmethods):
1.6319 + _span(span),
1.6320 + _verification_bm(verification_bm),
1.6321 + _cms_bm(cms_bm),
1.6322 + _should_do_nmethods(should_do_nmethods) {
1.6323 + assert(_ref_processor == NULL, "deliberately left NULL");
1.6324 + assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
1.6325 +}
1.6326 +
1.6327 +void MarkRefsIntoVerifyClosure::do_oop(oop* p) {
1.6328 + // if p points into _span, then mark corresponding bit in _markBitMap
1.6329 + oop this_oop = *p;
1.6330 + if (this_oop != NULL) {
1.6331 + assert(this_oop->is_oop(), "expected an oop");
1.6332 + HeapWord* addr = (HeapWord*)this_oop;
1.6333 + if (_span.contains(addr)) {
1.6334 + _verification_bm->mark(addr);
1.6335 + if (!_cms_bm->isMarked(addr)) {
1.6336 + oop(addr)->print();
1.6337 + gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", addr);
1.6338 + fatal("... aborting");
1.6339 + }
1.6340 + }
1.6341 + }
1.6342 +}
1.6343 +
1.6344 +//////////////////////////////////////////////////
1.6345 +// MarkRefsIntoAndScanClosure
1.6346 +//////////////////////////////////////////////////
1.6347 +
1.6348 +MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
1.6349 + ReferenceProcessor* rp,
1.6350 + CMSBitMap* bit_map,
1.6351 + CMSBitMap* mod_union_table,
1.6352 + CMSMarkStack* mark_stack,
1.6353 + CMSMarkStack* revisit_stack,
1.6354 + CMSCollector* collector,
1.6355 + bool should_yield,
1.6356 + bool concurrent_precleaning):
1.6357 + _collector(collector),
1.6358 + _span(span),
1.6359 + _bit_map(bit_map),
1.6360 + _mark_stack(mark_stack),
1.6361 + _pushAndMarkClosure(collector, span, rp, bit_map, mod_union_table,
1.6362 + mark_stack, revisit_stack, concurrent_precleaning),
1.6363 + _yield(should_yield),
1.6364 + _concurrent_precleaning(concurrent_precleaning),
1.6365 + _freelistLock(NULL)
1.6366 +{
1.6367 + _ref_processor = rp;
1.6368 + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
1.6369 +}
1.6370 +
1.6371 +// This closure is used to mark refs into the CMS generation at the
1.6372 +// second (final) checkpoint, and to scan and transitively follow
1.6373 +// the unmarked oops. It is also used during the concurrent precleaning
1.6374 +// phase while scanning objects on dirty cards in the CMS generation.
1.6375 +// The marks are made in the marking bit map and the marking stack is
1.6376 +// used for keeping the (newly) grey objects during the scan.
1.6377 +// The parallel version (Par_...) appears further below.
1.6378 +void MarkRefsIntoAndScanClosure::do_oop(oop* p) {
1.6379 + oop this_oop = *p;
1.6380 + if (this_oop != NULL) {
1.6381 + assert(this_oop->is_oop(), "expected an oop");
1.6382 + HeapWord* addr = (HeapWord*)this_oop;
1.6383 + assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
1.6384 + assert(_collector->overflow_list_is_empty(), "should be empty");
1.6385 + if (_span.contains(addr) &&
1.6386 + !_bit_map->isMarked(addr)) {
1.6387 + // mark bit map (object is now grey)
1.6388 + _bit_map->mark(addr);
1.6389 + // push on marking stack (stack should be empty), and drain the
1.6390 + // stack by applying this closure to the oops in the oops popped
1.6391 + // from the stack (i.e. blacken the grey objects)
1.6392 + bool res = _mark_stack->push(this_oop);
1.6393 + assert(res, "Should have space to push on empty stack");
1.6394 + do {
1.6395 + oop new_oop = _mark_stack->pop();
1.6396 + assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
1.6397 + assert(new_oop->is_parsable(), "Found unparsable oop");
1.6398 + assert(_bit_map->isMarked((HeapWord*)new_oop),
1.6399 + "only grey objects on this stack");
1.6400 + // iterate over the oops in this oop, marking and pushing
1.6401 + // the ones in CMS heap (i.e. in _span).
1.6402 + new_oop->oop_iterate(&_pushAndMarkClosure);
1.6403 + // check if it's time to yield
1.6404 + do_yield_check();
1.6405 + } while (!_mark_stack->isEmpty() ||
1.6406 + (!_concurrent_precleaning && take_from_overflow_list()));
1.6407 + // if marking stack is empty, and we are not doing this
1.6408 + // during precleaning, then check the overflow list
1.6409 + }
1.6410 + assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
1.6411 + assert(_collector->overflow_list_is_empty(),
1.6412 + "overflow list was drained above");
1.6413 + // We could restore evacuated mark words, if any, used for
1.6414 + // overflow list links here because the overflow list is
1.6415 + // provably empty here. That would reduce the maximum
1.6416 + // size requirements for preserved_{oop,mark}_stack.
1.6417 + // But we'll just postpone it until we are all done
1.6418 + // so we can just stream through.
1.6419 + if (!_concurrent_precleaning && CMSOverflowEarlyRestoration) {
1.6420 + _collector->restore_preserved_marks_if_any();
1.6421 + assert(_collector->no_preserved_marks(), "No preserved marks");
1.6422 + }
1.6423 + assert(!CMSOverflowEarlyRestoration || _collector->no_preserved_marks(),
1.6424 + "All preserved marks should have been restored above");
1.6425 + }
1.6426 +}
1.6427 +
1.6428 +void MarkRefsIntoAndScanClosure::do_yield_work() {
1.6429 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.6430 + "CMS thread should hold CMS token");
1.6431 + assert_lock_strong(_freelistLock);
1.6432 + assert_lock_strong(_bit_map->lock());
1.6433 + // relinquish the free_list_lock and bitMaplock()
1.6434 + _bit_map->lock()->unlock();
1.6435 + _freelistLock->unlock();
1.6436 + ConcurrentMarkSweepThread::desynchronize(true);
1.6437 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.6438 + _collector->stopTimer();
1.6439 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.6440 + if (PrintCMSStatistics != 0) {
1.6441 + _collector->incrementYields();
1.6442 + }
1.6443 + _collector->icms_wait();
1.6444 +
1.6445 + // See the comment in coordinator_yield()
1.6446 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.6447 + ConcurrentMarkSweepThread::should_yield() &&
1.6448 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.6449 + os::sleep(Thread::current(), 1, false);
1.6450 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.6451 + }
1.6452 +
1.6453 + ConcurrentMarkSweepThread::synchronize(true);
1.6454 + _freelistLock->lock_without_safepoint_check();
1.6455 + _bit_map->lock()->lock_without_safepoint_check();
1.6456 + _collector->startTimer();
1.6457 +}
1.6458 +
1.6459 +///////////////////////////////////////////////////////////
1.6460 +// Par_MarkRefsIntoAndScanClosure: a parallel version of
1.6461 +// MarkRefsIntoAndScanClosure
1.6462 +///////////////////////////////////////////////////////////
1.6463 +Par_MarkRefsIntoAndScanClosure::Par_MarkRefsIntoAndScanClosure(
1.6464 + CMSCollector* collector, MemRegion span, ReferenceProcessor* rp,
1.6465 + CMSBitMap* bit_map, OopTaskQueue* work_queue, CMSMarkStack* revisit_stack):
1.6466 + _span(span),
1.6467 + _bit_map(bit_map),
1.6468 + _work_queue(work_queue),
1.6469 + _low_water_mark(MIN2((uint)(work_queue->max_elems()/4),
1.6470 + (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads))),
1.6471 + _par_pushAndMarkClosure(collector, span, rp, bit_map, work_queue,
1.6472 + revisit_stack)
1.6473 +{
1.6474 + _ref_processor = rp;
1.6475 + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
1.6476 +}
1.6477 +
1.6478 +// This closure is used to mark refs into the CMS generation at the
1.6479 +// second (final) checkpoint, and to scan and transitively follow
1.6480 +// the unmarked oops. The marks are made in the marking bit map and
1.6481 +// the work_queue is used for keeping the (newly) grey objects during
1.6482 +// the scan phase whence they are also available for stealing by parallel
1.6483 +// threads. Since the marking bit map is shared, updates are
1.6484 +// synchronized (via CAS).
1.6485 +void Par_MarkRefsIntoAndScanClosure::do_oop(oop* p) {
1.6486 + oop this_oop = *p;
1.6487 + if (this_oop != NULL) {
1.6488 + // Ignore mark word because this could be an already marked oop
1.6489 + // that may be chained at the end of the overflow list.
1.6490 + assert(this_oop->is_oop(true /* ignore mark word */), "expected an oop");
1.6491 + HeapWord* addr = (HeapWord*)this_oop;
1.6492 + if (_span.contains(addr) &&
1.6493 + !_bit_map->isMarked(addr)) {
1.6494 + // mark bit map (object will become grey):
1.6495 + // It is possible for several threads to be
1.6496 + // trying to "claim" this object concurrently;
1.6497 + // the unique thread that succeeds in marking the
1.6498 + // object first will do the subsequent push on
1.6499 + // to the work queue (or overflow list).
1.6500 + if (_bit_map->par_mark(addr)) {
1.6501 + // push on work_queue (which may not be empty), and trim the
1.6502 + // queue to an appropriate length by applying this closure to
1.6503 + // the oops in the oops popped from the stack (i.e. blacken the
1.6504 + // grey objects)
1.6505 + bool res = _work_queue->push(this_oop);
1.6506 + assert(res, "Low water mark should be less than capacity?");
1.6507 + trim_queue(_low_water_mark);
1.6508 + } // Else, another thread claimed the object
1.6509 + }
1.6510 + }
1.6511 +}
1.6512 +
1.6513 +// This closure is used to rescan the marked objects on the dirty cards
1.6514 +// in the mod union table and the card table proper.
1.6515 +size_t ScanMarkedObjectsAgainCarefullyClosure::do_object_careful_m(
1.6516 + oop p, MemRegion mr) {
1.6517 +
1.6518 + size_t size = 0;
1.6519 + HeapWord* addr = (HeapWord*)p;
1.6520 + DEBUG_ONLY(_collector->verify_work_stacks_empty();)
1.6521 + assert(_span.contains(addr), "we are scanning the CMS generation");
1.6522 + // check if it's time to yield
1.6523 + if (do_yield_check()) {
1.6524 + // We yielded for some foreground stop-world work,
1.6525 + // and we have been asked to abort this ongoing preclean cycle.
1.6526 + return 0;
1.6527 + }
1.6528 + if (_bitMap->isMarked(addr)) {
1.6529 + // it's marked; is it potentially uninitialized?
1.6530 + if (p->klass() != NULL) {
1.6531 + if (CMSPermGenPrecleaningEnabled && !p->is_parsable()) {
1.6532 + // Signal precleaning to redirty the card since
1.6533 + // the klass pointer is already installed.
1.6534 + assert(size == 0, "Initial value");
1.6535 + } else {
1.6536 + assert(p->is_parsable(), "must be parsable.");
1.6537 + // an initialized object; ignore mark word in verification below
1.6538 + // since we are running concurrent with mutators
1.6539 + assert(p->is_oop(true), "should be an oop");
1.6540 + if (p->is_objArray()) {
1.6541 + // objArrays are precisely marked; restrict scanning
1.6542 + // to dirty cards only.
1.6543 + size = p->oop_iterate(_scanningClosure, mr);
1.6544 + assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
1.6545 + "adjustObjectSize should be the identity for array sizes, "
1.6546 + "which are necessarily larger than minimum object size of "
1.6547 + "two heap words");
1.6548 + } else {
1.6549 + // A non-array may have been imprecisely marked; we need
1.6550 + // to scan object in its entirety.
1.6551 + size = CompactibleFreeListSpace::adjustObjectSize(
1.6552 + p->oop_iterate(_scanningClosure));
1.6553 + }
1.6554 + #ifdef DEBUG
1.6555 + size_t direct_size =
1.6556 + CompactibleFreeListSpace::adjustObjectSize(p->size());
1.6557 + assert(size == direct_size, "Inconsistency in size");
1.6558 + assert(size >= 3, "Necessary for Printezis marks to work");
1.6559 + if (!_bitMap->isMarked(addr+1)) {
1.6560 + _bitMap->verifyNoOneBitsInRange(addr+2, addr+size);
1.6561 + } else {
1.6562 + _bitMap->verifyNoOneBitsInRange(addr+2, addr+size-1);
1.6563 + assert(_bitMap->isMarked(addr+size-1),
1.6564 + "inconsistent Printezis mark");
1.6565 + }
1.6566 + #endif // DEBUG
1.6567 + }
1.6568 + } else {
1.6569 + // an unitialized object
1.6570 + assert(_bitMap->isMarked(addr+1), "missing Printezis mark?");
1.6571 + HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
1.6572 + size = pointer_delta(nextOneAddr + 1, addr);
1.6573 + assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
1.6574 + "alignment problem");
1.6575 + // Note that pre-cleaning needn't redirty the card. OopDesc::set_klass()
1.6576 + // will dirty the card when the klass pointer is installed in the
1.6577 + // object (signalling the completion of initialization).
1.6578 + }
1.6579 + } else {
1.6580 + // Either a not yet marked object or an uninitialized object
1.6581 + if (p->klass() == NULL || !p->is_parsable()) {
1.6582 + // An uninitialized object, skip to the next card, since
1.6583 + // we may not be able to read its P-bits yet.
1.6584 + assert(size == 0, "Initial value");
1.6585 + } else {
1.6586 + // An object not (yet) reached by marking: we merely need to
1.6587 + // compute its size so as to go look at the next block.
1.6588 + assert(p->is_oop(true), "should be an oop");
1.6589 + size = CompactibleFreeListSpace::adjustObjectSize(p->size());
1.6590 + }
1.6591 + }
1.6592 + DEBUG_ONLY(_collector->verify_work_stacks_empty();)
1.6593 + return size;
1.6594 +}
1.6595 +
1.6596 +void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
1.6597 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.6598 + "CMS thread should hold CMS token");
1.6599 + assert_lock_strong(_freelistLock);
1.6600 + assert_lock_strong(_bitMap->lock());
1.6601 + // relinquish the free_list_lock and bitMaplock()
1.6602 + _bitMap->lock()->unlock();
1.6603 + _freelistLock->unlock();
1.6604 + ConcurrentMarkSweepThread::desynchronize(true);
1.6605 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.6606 + _collector->stopTimer();
1.6607 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.6608 + if (PrintCMSStatistics != 0) {
1.6609 + _collector->incrementYields();
1.6610 + }
1.6611 + _collector->icms_wait();
1.6612 +
1.6613 + // See the comment in coordinator_yield()
1.6614 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.6615 + ConcurrentMarkSweepThread::should_yield() &&
1.6616 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.6617 + os::sleep(Thread::current(), 1, false);
1.6618 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.6619 + }
1.6620 +
1.6621 + ConcurrentMarkSweepThread::synchronize(true);
1.6622 + _freelistLock->lock_without_safepoint_check();
1.6623 + _bitMap->lock()->lock_without_safepoint_check();
1.6624 + _collector->startTimer();
1.6625 +}
1.6626 +
1.6627 +
1.6628 +//////////////////////////////////////////////////////////////////
1.6629 +// SurvivorSpacePrecleanClosure
1.6630 +//////////////////////////////////////////////////////////////////
1.6631 +// This (single-threaded) closure is used to preclean the oops in
1.6632 +// the survivor spaces.
1.6633 +size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) {
1.6634 +
1.6635 + HeapWord* addr = (HeapWord*)p;
1.6636 + DEBUG_ONLY(_collector->verify_work_stacks_empty();)
1.6637 + assert(!_span.contains(addr), "we are scanning the survivor spaces");
1.6638 + assert(p->klass() != NULL, "object should be initializd");
1.6639 + assert(p->is_parsable(), "must be parsable.");
1.6640 + // an initialized object; ignore mark word in verification below
1.6641 + // since we are running concurrent with mutators
1.6642 + assert(p->is_oop(true), "should be an oop");
1.6643 + // Note that we do not yield while we iterate over
1.6644 + // the interior oops of p, pushing the relevant ones
1.6645 + // on our marking stack.
1.6646 + size_t size = p->oop_iterate(_scanning_closure);
1.6647 + do_yield_check();
1.6648 + // Observe that below, we do not abandon the preclean
1.6649 + // phase as soon as we should; rather we empty the
1.6650 + // marking stack before returning. This is to satisfy
1.6651 + // some existing assertions. In general, it may be a
1.6652 + // good idea to abort immediately and complete the marking
1.6653 + // from the grey objects at a later time.
1.6654 + while (!_mark_stack->isEmpty()) {
1.6655 + oop new_oop = _mark_stack->pop();
1.6656 + assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
1.6657 + assert(new_oop->is_parsable(), "Found unparsable oop");
1.6658 + assert(_bit_map->isMarked((HeapWord*)new_oop),
1.6659 + "only grey objects on this stack");
1.6660 + // iterate over the oops in this oop, marking and pushing
1.6661 + // the ones in CMS heap (i.e. in _span).
1.6662 + new_oop->oop_iterate(_scanning_closure);
1.6663 + // check if it's time to yield
1.6664 + do_yield_check();
1.6665 + }
1.6666 + unsigned int after_count =
1.6667 + GenCollectedHeap::heap()->total_collections();
1.6668 + bool abort = (_before_count != after_count) ||
1.6669 + _collector->should_abort_preclean();
1.6670 + return abort ? 0 : size;
1.6671 +}
1.6672 +
1.6673 +void SurvivorSpacePrecleanClosure::do_yield_work() {
1.6674 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.6675 + "CMS thread should hold CMS token");
1.6676 + assert_lock_strong(_bit_map->lock());
1.6677 + // Relinquish the bit map lock
1.6678 + _bit_map->lock()->unlock();
1.6679 + ConcurrentMarkSweepThread::desynchronize(true);
1.6680 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.6681 + _collector->stopTimer();
1.6682 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.6683 + if (PrintCMSStatistics != 0) {
1.6684 + _collector->incrementYields();
1.6685 + }
1.6686 + _collector->icms_wait();
1.6687 +
1.6688 + // See the comment in coordinator_yield()
1.6689 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.6690 + ConcurrentMarkSweepThread::should_yield() &&
1.6691 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.6692 + os::sleep(Thread::current(), 1, false);
1.6693 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.6694 + }
1.6695 +
1.6696 + ConcurrentMarkSweepThread::synchronize(true);
1.6697 + _bit_map->lock()->lock_without_safepoint_check();
1.6698 + _collector->startTimer();
1.6699 +}
1.6700 +
1.6701 +// This closure is used to rescan the marked objects on the dirty cards
1.6702 +// in the mod union table and the card table proper. In the parallel
1.6703 +// case, although the bitMap is shared, we do a single read so the
1.6704 +// isMarked() query is "safe".
1.6705 +bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) {
1.6706 + // Ignore mark word because we are running concurrent with mutators
1.6707 + assert(p->is_oop_or_null(true), "expected an oop or null");
1.6708 + HeapWord* addr = (HeapWord*)p;
1.6709 + assert(_span.contains(addr), "we are scanning the CMS generation");
1.6710 + bool is_obj_array = false;
1.6711 + #ifdef DEBUG
1.6712 + if (!_parallel) {
1.6713 + assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)");
1.6714 + assert(_collector->overflow_list_is_empty(),
1.6715 + "overflow list should be empty");
1.6716 +
1.6717 + }
1.6718 + #endif // DEBUG
1.6719 + if (_bit_map->isMarked(addr)) {
1.6720 + // Obj arrays are precisely marked, non-arrays are not;
1.6721 + // so we scan objArrays precisely and non-arrays in their
1.6722 + // entirety.
1.6723 + if (p->is_objArray()) {
1.6724 + is_obj_array = true;
1.6725 + if (_parallel) {
1.6726 + p->oop_iterate(_par_scan_closure, mr);
1.6727 + } else {
1.6728 + p->oop_iterate(_scan_closure, mr);
1.6729 + }
1.6730 + } else {
1.6731 + if (_parallel) {
1.6732 + p->oop_iterate(_par_scan_closure);
1.6733 + } else {
1.6734 + p->oop_iterate(_scan_closure);
1.6735 + }
1.6736 + }
1.6737 + }
1.6738 + #ifdef DEBUG
1.6739 + if (!_parallel) {
1.6740 + assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
1.6741 + assert(_collector->overflow_list_is_empty(),
1.6742 + "overflow list should be empty");
1.6743 +
1.6744 + }
1.6745 + #endif // DEBUG
1.6746 + return is_obj_array;
1.6747 +}
1.6748 +
1.6749 +MarkFromRootsClosure::MarkFromRootsClosure(CMSCollector* collector,
1.6750 + MemRegion span,
1.6751 + CMSBitMap* bitMap, CMSMarkStack* markStack,
1.6752 + CMSMarkStack* revisitStack,
1.6753 + bool should_yield, bool verifying):
1.6754 + _collector(collector),
1.6755 + _span(span),
1.6756 + _bitMap(bitMap),
1.6757 + _mut(&collector->_modUnionTable),
1.6758 + _markStack(markStack),
1.6759 + _revisitStack(revisitStack),
1.6760 + _yield(should_yield),
1.6761 + _skipBits(0)
1.6762 +{
1.6763 + assert(_markStack->isEmpty(), "stack should be empty");
1.6764 + _finger = _bitMap->startWord();
1.6765 + _threshold = _finger;
1.6766 + assert(_collector->_restart_addr == NULL, "Sanity check");
1.6767 + assert(_span.contains(_finger), "Out of bounds _finger?");
1.6768 + DEBUG_ONLY(_verifying = verifying;)
1.6769 +}
1.6770 +
1.6771 +void MarkFromRootsClosure::reset(HeapWord* addr) {
1.6772 + assert(_markStack->isEmpty(), "would cause duplicates on stack");
1.6773 + assert(_span.contains(addr), "Out of bounds _finger?");
1.6774 + _finger = addr;
1.6775 + _threshold = (HeapWord*)round_to(
1.6776 + (intptr_t)_finger, CardTableModRefBS::card_size);
1.6777 +}
1.6778 +
1.6779 +// Should revisit to see if this should be restructured for
1.6780 +// greater efficiency.
1.6781 +void MarkFromRootsClosure::do_bit(size_t offset) {
1.6782 + if (_skipBits > 0) {
1.6783 + _skipBits--;
1.6784 + return;
1.6785 + }
1.6786 + // convert offset into a HeapWord*
1.6787 + HeapWord* addr = _bitMap->startWord() + offset;
1.6788 + assert(_bitMap->endWord() && addr < _bitMap->endWord(),
1.6789 + "address out of range");
1.6790 + assert(_bitMap->isMarked(addr), "tautology");
1.6791 + if (_bitMap->isMarked(addr+1)) {
1.6792 + // this is an allocated but not yet initialized object
1.6793 + assert(_skipBits == 0, "tautology");
1.6794 + _skipBits = 2; // skip next two marked bits ("Printezis-marks")
1.6795 + oop p = oop(addr);
1.6796 + if (p->klass() == NULL || !p->is_parsable()) {
1.6797 + DEBUG_ONLY(if (!_verifying) {)
1.6798 + // We re-dirty the cards on which this object lies and increase
1.6799 + // the _threshold so that we'll come back to scan this object
1.6800 + // during the preclean or remark phase. (CMSCleanOnEnter)
1.6801 + if (CMSCleanOnEnter) {
1.6802 + size_t sz = _collector->block_size_using_printezis_bits(addr);
1.6803 + HeapWord* start_card_addr = (HeapWord*)round_down(
1.6804 + (intptr_t)addr, CardTableModRefBS::card_size);
1.6805 + HeapWord* end_card_addr = (HeapWord*)round_to(
1.6806 + (intptr_t)(addr+sz), CardTableModRefBS::card_size);
1.6807 + MemRegion redirty_range = MemRegion(start_card_addr, end_card_addr);
1.6808 + assert(!redirty_range.is_empty(), "Arithmetical tautology");
1.6809 + // Bump _threshold to end_card_addr; note that
1.6810 + // _threshold cannot possibly exceed end_card_addr, anyhow.
1.6811 + // This prevents future clearing of the card as the scan proceeds
1.6812 + // to the right.
1.6813 + assert(_threshold <= end_card_addr,
1.6814 + "Because we are just scanning into this object");
1.6815 + if (_threshold < end_card_addr) {
1.6816 + _threshold = end_card_addr;
1.6817 + }
1.6818 + if (p->klass() != NULL) {
1.6819 + // Redirty the range of cards...
1.6820 + _mut->mark_range(redirty_range);
1.6821 + } // ...else the setting of klass will dirty the card anyway.
1.6822 + }
1.6823 + DEBUG_ONLY(})
1.6824 + return;
1.6825 + }
1.6826 + }
1.6827 + scanOopsInOop(addr);
1.6828 +}
1.6829 +
1.6830 +// We take a break if we've been at this for a while,
1.6831 +// so as to avoid monopolizing the locks involved.
1.6832 +void MarkFromRootsClosure::do_yield_work() {
1.6833 + // First give up the locks, then yield, then re-lock
1.6834 + // We should probably use a constructor/destructor idiom to
1.6835 + // do this unlock/lock or modify the MutexUnlocker class to
1.6836 + // serve our purpose. XXX
1.6837 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.6838 + "CMS thread should hold CMS token");
1.6839 + assert_lock_strong(_bitMap->lock());
1.6840 + _bitMap->lock()->unlock();
1.6841 + ConcurrentMarkSweepThread::desynchronize(true);
1.6842 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.6843 + _collector->stopTimer();
1.6844 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.6845 + if (PrintCMSStatistics != 0) {
1.6846 + _collector->incrementYields();
1.6847 + }
1.6848 + _collector->icms_wait();
1.6849 +
1.6850 + // See the comment in coordinator_yield()
1.6851 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.6852 + ConcurrentMarkSweepThread::should_yield() &&
1.6853 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.6854 + os::sleep(Thread::current(), 1, false);
1.6855 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.6856 + }
1.6857 +
1.6858 + ConcurrentMarkSweepThread::synchronize(true);
1.6859 + _bitMap->lock()->lock_without_safepoint_check();
1.6860 + _collector->startTimer();
1.6861 +}
1.6862 +
1.6863 +void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) {
1.6864 + assert(_bitMap->isMarked(ptr), "expected bit to be set");
1.6865 + assert(_markStack->isEmpty(),
1.6866 + "should drain stack to limit stack usage");
1.6867 + // convert ptr to an oop preparatory to scanning
1.6868 + oop this_oop = oop(ptr);
1.6869 + // Ignore mark word in verification below, since we
1.6870 + // may be running concurrent with mutators.
1.6871 + assert(this_oop->is_oop(true), "should be an oop");
1.6872 + assert(_finger <= ptr, "_finger runneth ahead");
1.6873 + // advance the finger to right end of this object
1.6874 + _finger = ptr + this_oop->size();
1.6875 + assert(_finger > ptr, "we just incremented it above");
1.6876 + // On large heaps, it may take us some time to get through
1.6877 + // the marking phase (especially if running iCMS). During
1.6878 + // this time it's possible that a lot of mutations have
1.6879 + // accumulated in the card table and the mod union table --
1.6880 + // these mutation records are redundant until we have
1.6881 + // actually traced into the corresponding card.
1.6882 + // Here, we check whether advancing the finger would make
1.6883 + // us cross into a new card, and if so clear corresponding
1.6884 + // cards in the MUT (preclean them in the card-table in the
1.6885 + // future).
1.6886 +
1.6887 + DEBUG_ONLY(if (!_verifying) {)
1.6888 + // The clean-on-enter optimization is disabled by default,
1.6889 + // until we fix 6178663.
1.6890 + if (CMSCleanOnEnter && (_finger > _threshold)) {
1.6891 + // [_threshold, _finger) represents the interval
1.6892 + // of cards to be cleared in MUT (or precleaned in card table).
1.6893 + // The set of cards to be cleared is all those that overlap
1.6894 + // with the interval [_threshold, _finger); note that
1.6895 + // _threshold is always kept card-aligned but _finger isn't
1.6896 + // always card-aligned.
1.6897 + HeapWord* old_threshold = _threshold;
1.6898 + assert(old_threshold == (HeapWord*)round_to(
1.6899 + (intptr_t)old_threshold, CardTableModRefBS::card_size),
1.6900 + "_threshold should always be card-aligned");
1.6901 + _threshold = (HeapWord*)round_to(
1.6902 + (intptr_t)_finger, CardTableModRefBS::card_size);
1.6903 + MemRegion mr(old_threshold, _threshold);
1.6904 + assert(!mr.is_empty(), "Control point invariant");
1.6905 + assert(_span.contains(mr), "Should clear within span");
1.6906 + // XXX When _finger crosses from old gen into perm gen
1.6907 + // we may be doing unnecessary cleaning; do better in the
1.6908 + // future by detecting that condition and clearing fewer
1.6909 + // MUT/CT entries.
1.6910 + _mut->clear_range(mr);
1.6911 + }
1.6912 + DEBUG_ONLY(})
1.6913 +
1.6914 + // Note: the finger doesn't advance while we drain
1.6915 + // the stack below.
1.6916 + PushOrMarkClosure pushOrMarkClosure(_collector,
1.6917 + _span, _bitMap, _markStack,
1.6918 + _revisitStack,
1.6919 + _finger, this);
1.6920 + bool res = _markStack->push(this_oop);
1.6921 + assert(res, "Empty non-zero size stack should have space for single push");
1.6922 + while (!_markStack->isEmpty()) {
1.6923 + oop new_oop = _markStack->pop();
1.6924 + // Skip verifying header mark word below because we are
1.6925 + // running concurrent with mutators.
1.6926 + assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
1.6927 + // now scan this oop's oops
1.6928 + new_oop->oop_iterate(&pushOrMarkClosure);
1.6929 + do_yield_check();
1.6930 + }
1.6931 + assert(_markStack->isEmpty(), "tautology, emphasizing post-condition");
1.6932 +}
1.6933 +
1.6934 +Par_MarkFromRootsClosure::Par_MarkFromRootsClosure(CMSConcMarkingTask* task,
1.6935 + CMSCollector* collector, MemRegion span,
1.6936 + CMSBitMap* bit_map,
1.6937 + OopTaskQueue* work_queue,
1.6938 + CMSMarkStack* overflow_stack,
1.6939 + CMSMarkStack* revisit_stack,
1.6940 + bool should_yield):
1.6941 + _collector(collector),
1.6942 + _whole_span(collector->_span),
1.6943 + _span(span),
1.6944 + _bit_map(bit_map),
1.6945 + _mut(&collector->_modUnionTable),
1.6946 + _work_queue(work_queue),
1.6947 + _overflow_stack(overflow_stack),
1.6948 + _revisit_stack(revisit_stack),
1.6949 + _yield(should_yield),
1.6950 + _skip_bits(0),
1.6951 + _task(task)
1.6952 +{
1.6953 + assert(_work_queue->size() == 0, "work_queue should be empty");
1.6954 + _finger = span.start();
1.6955 + _threshold = _finger; // XXX Defer clear-on-enter optimization for now
1.6956 + assert(_span.contains(_finger), "Out of bounds _finger?");
1.6957 +}
1.6958 +
1.6959 +// Should revisit to see if this should be restructured for
1.6960 +// greater efficiency.
1.6961 +void Par_MarkFromRootsClosure::do_bit(size_t offset) {
1.6962 + if (_skip_bits > 0) {
1.6963 + _skip_bits--;
1.6964 + return;
1.6965 + }
1.6966 + // convert offset into a HeapWord*
1.6967 + HeapWord* addr = _bit_map->startWord() + offset;
1.6968 + assert(_bit_map->endWord() && addr < _bit_map->endWord(),
1.6969 + "address out of range");
1.6970 + assert(_bit_map->isMarked(addr), "tautology");
1.6971 + if (_bit_map->isMarked(addr+1)) {
1.6972 + // this is an allocated object that might not yet be initialized
1.6973 + assert(_skip_bits == 0, "tautology");
1.6974 + _skip_bits = 2; // skip next two marked bits ("Printezis-marks")
1.6975 + oop p = oop(addr);
1.6976 + if (p->klass() == NULL || !p->is_parsable()) {
1.6977 + // in the case of Clean-on-Enter optimization, redirty card
1.6978 + // and avoid clearing card by increasing the threshold.
1.6979 + return;
1.6980 + }
1.6981 + }
1.6982 + scan_oops_in_oop(addr);
1.6983 +}
1.6984 +
1.6985 +void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) {
1.6986 + assert(_bit_map->isMarked(ptr), "expected bit to be set");
1.6987 + // Should we assert that our work queue is empty or
1.6988 + // below some drain limit?
1.6989 + assert(_work_queue->size() == 0,
1.6990 + "should drain stack to limit stack usage");
1.6991 + // convert ptr to an oop preparatory to scanning
1.6992 + oop this_oop = oop(ptr);
1.6993 + // Ignore mark word in verification below, since we
1.6994 + // may be running concurrent with mutators.
1.6995 + assert(this_oop->is_oop(true), "should be an oop");
1.6996 + assert(_finger <= ptr, "_finger runneth ahead");
1.6997 + // advance the finger to right end of this object
1.6998 + _finger = ptr + this_oop->size();
1.6999 + assert(_finger > ptr, "we just incremented it above");
1.7000 + // On large heaps, it may take us some time to get through
1.7001 + // the marking phase (especially if running iCMS). During
1.7002 + // this time it's possible that a lot of mutations have
1.7003 + // accumulated in the card table and the mod union table --
1.7004 + // these mutation records are redundant until we have
1.7005 + // actually traced into the corresponding card.
1.7006 + // Here, we check whether advancing the finger would make
1.7007 + // us cross into a new card, and if so clear corresponding
1.7008 + // cards in the MUT (preclean them in the card-table in the
1.7009 + // future).
1.7010 +
1.7011 + // The clean-on-enter optimization is disabled by default,
1.7012 + // until we fix 6178663.
1.7013 + if (CMSCleanOnEnter && (_finger > _threshold)) {
1.7014 + // [_threshold, _finger) represents the interval
1.7015 + // of cards to be cleared in MUT (or precleaned in card table).
1.7016 + // The set of cards to be cleared is all those that overlap
1.7017 + // with the interval [_threshold, _finger); note that
1.7018 + // _threshold is always kept card-aligned but _finger isn't
1.7019 + // always card-aligned.
1.7020 + HeapWord* old_threshold = _threshold;
1.7021 + assert(old_threshold == (HeapWord*)round_to(
1.7022 + (intptr_t)old_threshold, CardTableModRefBS::card_size),
1.7023 + "_threshold should always be card-aligned");
1.7024 + _threshold = (HeapWord*)round_to(
1.7025 + (intptr_t)_finger, CardTableModRefBS::card_size);
1.7026 + MemRegion mr(old_threshold, _threshold);
1.7027 + assert(!mr.is_empty(), "Control point invariant");
1.7028 + assert(_span.contains(mr), "Should clear within span"); // _whole_span ??
1.7029 + // XXX When _finger crosses from old gen into perm gen
1.7030 + // we may be doing unnecessary cleaning; do better in the
1.7031 + // future by detecting that condition and clearing fewer
1.7032 + // MUT/CT entries.
1.7033 + _mut->clear_range(mr);
1.7034 + }
1.7035 +
1.7036 + // Note: the local finger doesn't advance while we drain
1.7037 + // the stack below, but the global finger sure can and will.
1.7038 + HeapWord** gfa = _task->global_finger_addr();
1.7039 + Par_PushOrMarkClosure pushOrMarkClosure(_collector,
1.7040 + _span, _bit_map,
1.7041 + _work_queue,
1.7042 + _overflow_stack,
1.7043 + _revisit_stack,
1.7044 + _finger,
1.7045 + gfa, this);
1.7046 + bool res = _work_queue->push(this_oop); // overflow could occur here
1.7047 + assert(res, "Will hold once we use workqueues");
1.7048 + while (true) {
1.7049 + oop new_oop;
1.7050 + if (!_work_queue->pop_local(new_oop)) {
1.7051 + // We emptied our work_queue; check if there's stuff that can
1.7052 + // be gotten from the overflow stack.
1.7053 + if (CMSConcMarkingTask::get_work_from_overflow_stack(
1.7054 + _overflow_stack, _work_queue)) {
1.7055 + do_yield_check();
1.7056 + continue;
1.7057 + } else { // done
1.7058 + break;
1.7059 + }
1.7060 + }
1.7061 + // Skip verifying header mark word below because we are
1.7062 + // running concurrent with mutators.
1.7063 + assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
1.7064 + // now scan this oop's oops
1.7065 + new_oop->oop_iterate(&pushOrMarkClosure);
1.7066 + do_yield_check();
1.7067 + }
1.7068 + assert(_work_queue->size() == 0, "tautology, emphasizing post-condition");
1.7069 +}
1.7070 +
1.7071 +// Yield in response to a request from VM Thread or
1.7072 +// from mutators.
1.7073 +void Par_MarkFromRootsClosure::do_yield_work() {
1.7074 + assert(_task != NULL, "sanity");
1.7075 + _task->yield();
1.7076 +}
1.7077 +
1.7078 +// A variant of the above used for verifying CMS marking work.
1.7079 +MarkFromRootsVerifyClosure::MarkFromRootsVerifyClosure(CMSCollector* collector,
1.7080 + MemRegion span,
1.7081 + CMSBitMap* verification_bm, CMSBitMap* cms_bm,
1.7082 + CMSMarkStack* mark_stack):
1.7083 + _collector(collector),
1.7084 + _span(span),
1.7085 + _verification_bm(verification_bm),
1.7086 + _cms_bm(cms_bm),
1.7087 + _mark_stack(mark_stack),
1.7088 + _pam_verify_closure(collector, span, verification_bm, cms_bm,
1.7089 + mark_stack)
1.7090 +{
1.7091 + assert(_mark_stack->isEmpty(), "stack should be empty");
1.7092 + _finger = _verification_bm->startWord();
1.7093 + assert(_collector->_restart_addr == NULL, "Sanity check");
1.7094 + assert(_span.contains(_finger), "Out of bounds _finger?");
1.7095 +}
1.7096 +
1.7097 +void MarkFromRootsVerifyClosure::reset(HeapWord* addr) {
1.7098 + assert(_mark_stack->isEmpty(), "would cause duplicates on stack");
1.7099 + assert(_span.contains(addr), "Out of bounds _finger?");
1.7100 + _finger = addr;
1.7101 +}
1.7102 +
1.7103 +// Should revisit to see if this should be restructured for
1.7104 +// greater efficiency.
1.7105 +void MarkFromRootsVerifyClosure::do_bit(size_t offset) {
1.7106 + // convert offset into a HeapWord*
1.7107 + HeapWord* addr = _verification_bm->startWord() + offset;
1.7108 + assert(_verification_bm->endWord() && addr < _verification_bm->endWord(),
1.7109 + "address out of range");
1.7110 + assert(_verification_bm->isMarked(addr), "tautology");
1.7111 + assert(_cms_bm->isMarked(addr), "tautology");
1.7112 +
1.7113 + assert(_mark_stack->isEmpty(),
1.7114 + "should drain stack to limit stack usage");
1.7115 + // convert addr to an oop preparatory to scanning
1.7116 + oop this_oop = oop(addr);
1.7117 + assert(this_oop->is_oop(), "should be an oop");
1.7118 + assert(_finger <= addr, "_finger runneth ahead");
1.7119 + // advance the finger to right end of this object
1.7120 + _finger = addr + this_oop->size();
1.7121 + assert(_finger > addr, "we just incremented it above");
1.7122 + // Note: the finger doesn't advance while we drain
1.7123 + // the stack below.
1.7124 + bool res = _mark_stack->push(this_oop);
1.7125 + assert(res, "Empty non-zero size stack should have space for single push");
1.7126 + while (!_mark_stack->isEmpty()) {
1.7127 + oop new_oop = _mark_stack->pop();
1.7128 + assert(new_oop->is_oop(), "Oops! expected to pop an oop");
1.7129 + // now scan this oop's oops
1.7130 + new_oop->oop_iterate(&_pam_verify_closure);
1.7131 + }
1.7132 + assert(_mark_stack->isEmpty(), "tautology, emphasizing post-condition");
1.7133 +}
1.7134 +
1.7135 +PushAndMarkVerifyClosure::PushAndMarkVerifyClosure(
1.7136 + CMSCollector* collector, MemRegion span,
1.7137 + CMSBitMap* verification_bm, CMSBitMap* cms_bm,
1.7138 + CMSMarkStack* mark_stack):
1.7139 + OopClosure(collector->ref_processor()),
1.7140 + _collector(collector),
1.7141 + _span(span),
1.7142 + _verification_bm(verification_bm),
1.7143 + _cms_bm(cms_bm),
1.7144 + _mark_stack(mark_stack)
1.7145 +{ }
1.7146 +
1.7147 +
1.7148 +// Upon stack overflow, we discard (part of) the stack,
1.7149 +// remembering the least address amongst those discarded
1.7150 +// in CMSCollector's _restart_address.
1.7151 +void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
1.7152 + // Remember the least grey address discarded
1.7153 + HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
1.7154 + _collector->lower_restart_addr(ra);
1.7155 + _mark_stack->reset(); // discard stack contents
1.7156 + _mark_stack->expand(); // expand the stack if possible
1.7157 +}
1.7158 +
1.7159 +void PushAndMarkVerifyClosure::do_oop(oop* p) {
1.7160 + oop this_oop = *p;
1.7161 + assert(this_oop->is_oop_or_null(), "expected an oop or NULL");
1.7162 + HeapWord* addr = (HeapWord*)this_oop;
1.7163 + if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
1.7164 + // Oop lies in _span and isn't yet grey or black
1.7165 + _verification_bm->mark(addr); // now grey
1.7166 + if (!_cms_bm->isMarked(addr)) {
1.7167 + oop(addr)->print();
1.7168 + gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", addr);
1.7169 + fatal("... aborting");
1.7170 + }
1.7171 +
1.7172 + if (!_mark_stack->push(this_oop)) { // stack overflow
1.7173 + if (PrintCMSStatistics != 0) {
1.7174 + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
1.7175 + SIZE_FORMAT, _mark_stack->capacity());
1.7176 + }
1.7177 + assert(_mark_stack->isFull(), "Else push should have succeeded");
1.7178 + handle_stack_overflow(addr);
1.7179 + }
1.7180 + // anything including and to the right of _finger
1.7181 + // will be scanned as we iterate over the remainder of the
1.7182 + // bit map
1.7183 + }
1.7184 +}
1.7185 +
1.7186 +PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
1.7187 + MemRegion span,
1.7188 + CMSBitMap* bitMap, CMSMarkStack* markStack,
1.7189 + CMSMarkStack* revisitStack,
1.7190 + HeapWord* finger, MarkFromRootsClosure* parent) :
1.7191 + OopClosure(collector->ref_processor()),
1.7192 + _collector(collector),
1.7193 + _span(span),
1.7194 + _bitMap(bitMap),
1.7195 + _markStack(markStack),
1.7196 + _revisitStack(revisitStack),
1.7197 + _finger(finger),
1.7198 + _parent(parent),
1.7199 + _should_remember_klasses(collector->cms_should_unload_classes())
1.7200 +{ }
1.7201 +
1.7202 +Par_PushOrMarkClosure::Par_PushOrMarkClosure(CMSCollector* collector,
1.7203 + MemRegion span,
1.7204 + CMSBitMap* bit_map,
1.7205 + OopTaskQueue* work_queue,
1.7206 + CMSMarkStack* overflow_stack,
1.7207 + CMSMarkStack* revisit_stack,
1.7208 + HeapWord* finger,
1.7209 + HeapWord** global_finger_addr,
1.7210 + Par_MarkFromRootsClosure* parent) :
1.7211 + OopClosure(collector->ref_processor()),
1.7212 + _collector(collector),
1.7213 + _whole_span(collector->_span),
1.7214 + _span(span),
1.7215 + _bit_map(bit_map),
1.7216 + _work_queue(work_queue),
1.7217 + _overflow_stack(overflow_stack),
1.7218 + _revisit_stack(revisit_stack),
1.7219 + _finger(finger),
1.7220 + _global_finger_addr(global_finger_addr),
1.7221 + _parent(parent),
1.7222 + _should_remember_klasses(collector->cms_should_unload_classes())
1.7223 +{ }
1.7224 +
1.7225 +
1.7226 +void CMSCollector::lower_restart_addr(HeapWord* low) {
1.7227 + assert(_span.contains(low), "Out of bounds addr");
1.7228 + if (_restart_addr == NULL) {
1.7229 + _restart_addr = low;
1.7230 + } else {
1.7231 + _restart_addr = MIN2(_restart_addr, low);
1.7232 + }
1.7233 +}
1.7234 +
1.7235 +// Upon stack overflow, we discard (part of) the stack,
1.7236 +// remembering the least address amongst those discarded
1.7237 +// in CMSCollector's _restart_address.
1.7238 +void PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
1.7239 + // Remember the least grey address discarded
1.7240 + HeapWord* ra = (HeapWord*)_markStack->least_value(lost);
1.7241 + _collector->lower_restart_addr(ra);
1.7242 + _markStack->reset(); // discard stack contents
1.7243 + _markStack->expand(); // expand the stack if possible
1.7244 +}
1.7245 +
1.7246 +// Upon stack overflow, we discard (part of) the stack,
1.7247 +// remembering the least address amongst those discarded
1.7248 +// in CMSCollector's _restart_address.
1.7249 +void Par_PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
1.7250 + // We need to do this under a mutex to prevent other
1.7251 + // workers from interfering with the expansion below.
1.7252 + MutexLockerEx ml(_overflow_stack->par_lock(),
1.7253 + Mutex::_no_safepoint_check_flag);
1.7254 + // Remember the least grey address discarded
1.7255 + HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost);
1.7256 + _collector->lower_restart_addr(ra);
1.7257 + _overflow_stack->reset(); // discard stack contents
1.7258 + _overflow_stack->expand(); // expand the stack if possible
1.7259 +}
1.7260 +
1.7261 +
1.7262 +void PushOrMarkClosure::do_oop(oop* p) {
1.7263 + oop thisOop = *p;
1.7264 + // Ignore mark word because we are running concurrent with mutators.
1.7265 + assert(thisOop->is_oop_or_null(true), "expected an oop or NULL");
1.7266 + HeapWord* addr = (HeapWord*)thisOop;
1.7267 + if (_span.contains(addr) && !_bitMap->isMarked(addr)) {
1.7268 + // Oop lies in _span and isn't yet grey or black
1.7269 + _bitMap->mark(addr); // now grey
1.7270 + if (addr < _finger) {
1.7271 + // the bit map iteration has already either passed, or
1.7272 + // sampled, this bit in the bit map; we'll need to
1.7273 + // use the marking stack to scan this oop's oops.
1.7274 + bool simulate_overflow = false;
1.7275 + NOT_PRODUCT(
1.7276 + if (CMSMarkStackOverflowALot &&
1.7277 + _collector->simulate_overflow()) {
1.7278 + // simulate a stack overflow
1.7279 + simulate_overflow = true;
1.7280 + }
1.7281 + )
1.7282 + if (simulate_overflow || !_markStack->push(thisOop)) { // stack overflow
1.7283 + if (PrintCMSStatistics != 0) {
1.7284 + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
1.7285 + SIZE_FORMAT, _markStack->capacity());
1.7286 + }
1.7287 + assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded");
1.7288 + handle_stack_overflow(addr);
1.7289 + }
1.7290 + }
1.7291 + // anything including and to the right of _finger
1.7292 + // will be scanned as we iterate over the remainder of the
1.7293 + // bit map
1.7294 + do_yield_check();
1.7295 + }
1.7296 +}
1.7297 +
1.7298 +void Par_PushOrMarkClosure::do_oop(oop* p) {
1.7299 + oop this_oop = *p;
1.7300 + // Ignore mark word because we are running concurrent with mutators.
1.7301 + assert(this_oop->is_oop_or_null(true), "expected an oop or NULL");
1.7302 + HeapWord* addr = (HeapWord*)this_oop;
1.7303 + if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) {
1.7304 + // Oop lies in _span and isn't yet grey or black
1.7305 + // We read the global_finger (volatile read) strictly after marking oop
1.7306 + bool res = _bit_map->par_mark(addr); // now grey
1.7307 + volatile HeapWord** gfa = (volatile HeapWord**)_global_finger_addr;
1.7308 + // Should we push this marked oop on our stack?
1.7309 + // -- if someone else marked it, nothing to do
1.7310 + // -- if target oop is above global finger nothing to do
1.7311 + // -- if target oop is in chunk and above local finger
1.7312 + // then nothing to do
1.7313 + // -- else push on work queue
1.7314 + if ( !res // someone else marked it, they will deal with it
1.7315 + || (addr >= *gfa) // will be scanned in a later task
1.7316 + || (_span.contains(addr) && addr >= _finger)) { // later in this chunk
1.7317 + return;
1.7318 + }
1.7319 + // the bit map iteration has already either passed, or
1.7320 + // sampled, this bit in the bit map; we'll need to
1.7321 + // use the marking stack to scan this oop's oops.
1.7322 + bool simulate_overflow = false;
1.7323 + NOT_PRODUCT(
1.7324 + if (CMSMarkStackOverflowALot &&
1.7325 + _collector->simulate_overflow()) {
1.7326 + // simulate a stack overflow
1.7327 + simulate_overflow = true;
1.7328 + }
1.7329 + )
1.7330 + if (simulate_overflow ||
1.7331 + !(_work_queue->push(this_oop) || _overflow_stack->par_push(this_oop))) {
1.7332 + // stack overflow
1.7333 + if (PrintCMSStatistics != 0) {
1.7334 + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
1.7335 + SIZE_FORMAT, _overflow_stack->capacity());
1.7336 + }
1.7337 + // We cannot assert that the overflow stack is full because
1.7338 + // it may have been emptied since.
1.7339 + assert(simulate_overflow ||
1.7340 + _work_queue->size() == _work_queue->max_elems(),
1.7341 + "Else push should have succeeded");
1.7342 + handle_stack_overflow(addr);
1.7343 + }
1.7344 + do_yield_check();
1.7345 + }
1.7346 +}
1.7347 +
1.7348 +
1.7349 +PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector,
1.7350 + MemRegion span,
1.7351 + ReferenceProcessor* rp,
1.7352 + CMSBitMap* bit_map,
1.7353 + CMSBitMap* mod_union_table,
1.7354 + CMSMarkStack* mark_stack,
1.7355 + CMSMarkStack* revisit_stack,
1.7356 + bool concurrent_precleaning):
1.7357 + OopClosure(rp),
1.7358 + _collector(collector),
1.7359 + _span(span),
1.7360 + _bit_map(bit_map),
1.7361 + _mod_union_table(mod_union_table),
1.7362 + _mark_stack(mark_stack),
1.7363 + _revisit_stack(revisit_stack),
1.7364 + _concurrent_precleaning(concurrent_precleaning),
1.7365 + _should_remember_klasses(collector->cms_should_unload_classes())
1.7366 +{
1.7367 + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
1.7368 +}
1.7369 +
1.7370 +// Grey object rescan during pre-cleaning and second checkpoint phases --
1.7371 +// the non-parallel version (the parallel version appears further below.)
1.7372 +void PushAndMarkClosure::do_oop(oop* p) {
1.7373 + oop this_oop = *p;
1.7374 + // Ignore mark word verification. If during concurrent precleaning
1.7375 + // the object monitor may be locked. If during the checkpoint
1.7376 + // phases, the object may already have been reached by a different
1.7377 + // path and may be at the end of the global overflow list (so
1.7378 + // the mark word may be NULL).
1.7379 + assert(this_oop->is_oop_or_null(true/* ignore mark word */),
1.7380 + "expected an oop or NULL");
1.7381 + HeapWord* addr = (HeapWord*)this_oop;
1.7382 + // Check if oop points into the CMS generation
1.7383 + // and is not marked
1.7384 + if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
1.7385 + // a white object ...
1.7386 + _bit_map->mark(addr); // ... now grey
1.7387 + // push on the marking stack (grey set)
1.7388 + bool simulate_overflow = false;
1.7389 + NOT_PRODUCT(
1.7390 + if (CMSMarkStackOverflowALot &&
1.7391 + _collector->simulate_overflow()) {
1.7392 + // simulate a stack overflow
1.7393 + simulate_overflow = true;
1.7394 + }
1.7395 + )
1.7396 + if (simulate_overflow || !_mark_stack->push(this_oop)) {
1.7397 + if (_concurrent_precleaning) {
1.7398 + // During precleaning we can just dirty the appropriate card
1.7399 + // in the mod union table, thus ensuring that the object remains
1.7400 + // in the grey set and continue. Note that no one can be intefering
1.7401 + // with us in this action of dirtying the mod union table, so
1.7402 + // no locking is required.
1.7403 + _mod_union_table->mark(addr);
1.7404 + _collector->_ser_pmc_preclean_ovflw++;
1.7405 + } else {
1.7406 + // During the remark phase, we need to remember this oop
1.7407 + // in the overflow list.
1.7408 + _collector->push_on_overflow_list(this_oop);
1.7409 + _collector->_ser_pmc_remark_ovflw++;
1.7410 + }
1.7411 + }
1.7412 + }
1.7413 +}
1.7414 +
1.7415 +Par_PushAndMarkClosure::Par_PushAndMarkClosure(CMSCollector* collector,
1.7416 + MemRegion span,
1.7417 + ReferenceProcessor* rp,
1.7418 + CMSBitMap* bit_map,
1.7419 + OopTaskQueue* work_queue,
1.7420 + CMSMarkStack* revisit_stack):
1.7421 + OopClosure(rp),
1.7422 + _collector(collector),
1.7423 + _span(span),
1.7424 + _bit_map(bit_map),
1.7425 + _work_queue(work_queue),
1.7426 + _revisit_stack(revisit_stack),
1.7427 + _should_remember_klasses(collector->cms_should_unload_classes())
1.7428 +{
1.7429 + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
1.7430 +}
1.7431 +
1.7432 +// Grey object rescan during second checkpoint phase --
1.7433 +// the parallel version.
1.7434 +void Par_PushAndMarkClosure::do_oop(oop* p) {
1.7435 + oop this_oop = *p;
1.7436 + // In the assert below, we ignore the mark word because
1.7437 + // this oop may point to an already visited object that is
1.7438 + // on the overflow stack (in which case the mark word has
1.7439 + // been hijacked for chaining into the overflow stack --
1.7440 + // if this is the last object in the overflow stack then
1.7441 + // its mark word will be NULL). Because this object may
1.7442 + // have been subsequently popped off the global overflow
1.7443 + // stack, and the mark word possibly restored to the prototypical
1.7444 + // value, by the time we get to examined this failing assert in
1.7445 + // the debugger, is_oop_or_null(false) may subsequently start
1.7446 + // to hold.
1.7447 + assert(this_oop->is_oop_or_null(true),
1.7448 + "expected an oop or NULL");
1.7449 + HeapWord* addr = (HeapWord*)this_oop;
1.7450 + // Check if oop points into the CMS generation
1.7451 + // and is not marked
1.7452 + if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
1.7453 + // a white object ...
1.7454 + // If we manage to "claim" the object, by being the
1.7455 + // first thread to mark it, then we push it on our
1.7456 + // marking stack
1.7457 + if (_bit_map->par_mark(addr)) { // ... now grey
1.7458 + // push on work queue (grey set)
1.7459 + bool simulate_overflow = false;
1.7460 + NOT_PRODUCT(
1.7461 + if (CMSMarkStackOverflowALot &&
1.7462 + _collector->par_simulate_overflow()) {
1.7463 + // simulate a stack overflow
1.7464 + simulate_overflow = true;
1.7465 + }
1.7466 + )
1.7467 + if (simulate_overflow || !_work_queue->push(this_oop)) {
1.7468 + _collector->par_push_on_overflow_list(this_oop);
1.7469 + _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS
1.7470 + }
1.7471 + } // Else, some other thread got there first
1.7472 + }
1.7473 +}
1.7474 +
1.7475 +void PushAndMarkClosure::remember_klass(Klass* k) {
1.7476 + if (!_revisit_stack->push(oop(k))) {
1.7477 + fatal("Revisit stack overflowed in PushAndMarkClosure");
1.7478 + }
1.7479 +}
1.7480 +
1.7481 +void Par_PushAndMarkClosure::remember_klass(Klass* k) {
1.7482 + if (!_revisit_stack->par_push(oop(k))) {
1.7483 + fatal("Revist stack overflowed in Par_PushAndMarkClosure");
1.7484 + }
1.7485 +}
1.7486 +
1.7487 +void CMSPrecleanRefsYieldClosure::do_yield_work() {
1.7488 + Mutex* bml = _collector->bitMapLock();
1.7489 + assert_lock_strong(bml);
1.7490 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.7491 + "CMS thread should hold CMS token");
1.7492 +
1.7493 + bml->unlock();
1.7494 + ConcurrentMarkSweepThread::desynchronize(true);
1.7495 +
1.7496 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.7497 +
1.7498 + _collector->stopTimer();
1.7499 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.7500 + if (PrintCMSStatistics != 0) {
1.7501 + _collector->incrementYields();
1.7502 + }
1.7503 + _collector->icms_wait();
1.7504 +
1.7505 + // See the comment in coordinator_yield()
1.7506 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.7507 + ConcurrentMarkSweepThread::should_yield() &&
1.7508 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.7509 + os::sleep(Thread::current(), 1, false);
1.7510 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.7511 + }
1.7512 +
1.7513 + ConcurrentMarkSweepThread::synchronize(true);
1.7514 + bml->lock();
1.7515 +
1.7516 + _collector->startTimer();
1.7517 +}
1.7518 +
1.7519 +bool CMSPrecleanRefsYieldClosure::should_return() {
1.7520 + if (ConcurrentMarkSweepThread::should_yield()) {
1.7521 + do_yield_work();
1.7522 + }
1.7523 + return _collector->foregroundGCIsActive();
1.7524 +}
1.7525 +
1.7526 +void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
1.7527 + assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
1.7528 + "mr should be aligned to start at a card boundary");
1.7529 + // We'd like to assert:
1.7530 + // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
1.7531 + // "mr should be a range of cards");
1.7532 + // However, that would be too strong in one case -- the last
1.7533 + // partition ends at _unallocated_block which, in general, can be
1.7534 + // an arbitrary boundary, not necessarily card aligned.
1.7535 + if (PrintCMSStatistics != 0) {
1.7536 + _num_dirty_cards +=
1.7537 + mr.word_size()/CardTableModRefBS::card_size_in_words;
1.7538 + }
1.7539 + _space->object_iterate_mem(mr, &_scan_cl);
1.7540 +}
1.7541 +
1.7542 +SweepClosure::SweepClosure(CMSCollector* collector,
1.7543 + ConcurrentMarkSweepGeneration* g,
1.7544 + CMSBitMap* bitMap, bool should_yield) :
1.7545 + _collector(collector),
1.7546 + _g(g),
1.7547 + _sp(g->cmsSpace()),
1.7548 + _limit(_sp->sweep_limit()),
1.7549 + _freelistLock(_sp->freelistLock()),
1.7550 + _bitMap(bitMap),
1.7551 + _yield(should_yield),
1.7552 + _inFreeRange(false), // No free range at beginning of sweep
1.7553 + _freeRangeInFreeLists(false), // No free range at beginning of sweep
1.7554 + _lastFreeRangeCoalesced(false),
1.7555 + _freeFinger(g->used_region().start())
1.7556 +{
1.7557 + NOT_PRODUCT(
1.7558 + _numObjectsFreed = 0;
1.7559 + _numWordsFreed = 0;
1.7560 + _numObjectsLive = 0;
1.7561 + _numWordsLive = 0;
1.7562 + _numObjectsAlreadyFree = 0;
1.7563 + _numWordsAlreadyFree = 0;
1.7564 + _last_fc = NULL;
1.7565 +
1.7566 + _sp->initializeIndexedFreeListArrayReturnedBytes();
1.7567 + _sp->dictionary()->initializeDictReturnedBytes();
1.7568 + )
1.7569 + assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
1.7570 + "sweep _limit out of bounds");
1.7571 + if (CMSTraceSweeper) {
1.7572 + gclog_or_tty->print("\n====================\nStarting new sweep\n");
1.7573 + }
1.7574 +}
1.7575 +
1.7576 +// We need this destructor to reclaim any space at the end
1.7577 +// of the space, which do_blk below may not have added back to
1.7578 +// the free lists. [basically dealing with the "fringe effect"]
1.7579 +SweepClosure::~SweepClosure() {
1.7580 + assert_lock_strong(_freelistLock);
1.7581 + // this should be treated as the end of a free run if any
1.7582 + // The current free range should be returned to the free lists
1.7583 + // as one coalesced chunk.
1.7584 + if (inFreeRange()) {
1.7585 + flushCurFreeChunk(freeFinger(),
1.7586 + pointer_delta(_limit, freeFinger()));
1.7587 + assert(freeFinger() < _limit, "the finger pointeth off base");
1.7588 + if (CMSTraceSweeper) {
1.7589 + gclog_or_tty->print("destructor:");
1.7590 + gclog_or_tty->print("Sweep:put_free_blk 0x%x ("SIZE_FORMAT") "
1.7591 + "[coalesced:"SIZE_FORMAT"]\n",
1.7592 + freeFinger(), pointer_delta(_limit, freeFinger()),
1.7593 + lastFreeRangeCoalesced());
1.7594 + }
1.7595 + }
1.7596 + NOT_PRODUCT(
1.7597 + if (Verbose && PrintGC) {
1.7598 + gclog_or_tty->print("Collected "SIZE_FORMAT" objects, "
1.7599 + SIZE_FORMAT " bytes",
1.7600 + _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
1.7601 + gclog_or_tty->print_cr("\nLive "SIZE_FORMAT" objects, "
1.7602 + SIZE_FORMAT" bytes "
1.7603 + "Already free "SIZE_FORMAT" objects, "SIZE_FORMAT" bytes",
1.7604 + _numObjectsLive, _numWordsLive*sizeof(HeapWord),
1.7605 + _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
1.7606 + size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree) *
1.7607 + sizeof(HeapWord);
1.7608 + gclog_or_tty->print_cr("Total sweep: "SIZE_FORMAT" bytes", totalBytes);
1.7609 +
1.7610 + if (PrintCMSStatistics && CMSVerifyReturnedBytes) {
1.7611 + size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
1.7612 + size_t dictReturnedBytes = _sp->dictionary()->sumDictReturnedBytes();
1.7613 + size_t returnedBytes = indexListReturnedBytes + dictReturnedBytes;
1.7614 + gclog_or_tty->print("Returned "SIZE_FORMAT" bytes", returnedBytes);
1.7615 + gclog_or_tty->print(" Indexed List Returned "SIZE_FORMAT" bytes",
1.7616 + indexListReturnedBytes);
1.7617 + gclog_or_tty->print_cr(" Dictionary Returned "SIZE_FORMAT" bytes",
1.7618 + dictReturnedBytes);
1.7619 + }
1.7620 + }
1.7621 + )
1.7622 + // Now, in debug mode, just null out the sweep_limit
1.7623 + NOT_PRODUCT(_sp->clear_sweep_limit();)
1.7624 + if (CMSTraceSweeper) {
1.7625 + gclog_or_tty->print("end of sweep\n================\n");
1.7626 + }
1.7627 +}
1.7628 +
1.7629 +void SweepClosure::initialize_free_range(HeapWord* freeFinger,
1.7630 + bool freeRangeInFreeLists) {
1.7631 + if (CMSTraceSweeper) {
1.7632 + gclog_or_tty->print("---- Start free range 0x%x with free block [%d] (%d)\n",
1.7633 + freeFinger, _sp->block_size(freeFinger),
1.7634 + freeRangeInFreeLists);
1.7635 + }
1.7636 + assert(!inFreeRange(), "Trampling existing free range");
1.7637 + set_inFreeRange(true);
1.7638 + set_lastFreeRangeCoalesced(false);
1.7639 +
1.7640 + set_freeFinger(freeFinger);
1.7641 + set_freeRangeInFreeLists(freeRangeInFreeLists);
1.7642 + if (CMSTestInFreeList) {
1.7643 + if (freeRangeInFreeLists) {
1.7644 + FreeChunk* fc = (FreeChunk*) freeFinger;
1.7645 + assert(fc->isFree(), "A chunk on the free list should be free.");
1.7646 + assert(fc->size() > 0, "Free range should have a size");
1.7647 + assert(_sp->verifyChunkInFreeLists(fc), "Chunk is not in free lists");
1.7648 + }
1.7649 + }
1.7650 +}
1.7651 +
1.7652 +// Note that the sweeper runs concurrently with mutators. Thus,
1.7653 +// it is possible for direct allocation in this generation to happen
1.7654 +// in the middle of the sweep. Note that the sweeper also coalesces
1.7655 +// contiguous free blocks. Thus, unless the sweeper and the allocator
1.7656 +// synchronize appropriately freshly allocated blocks may get swept up.
1.7657 +// This is accomplished by the sweeper locking the free lists while
1.7658 +// it is sweeping. Thus blocks that are determined to be free are
1.7659 +// indeed free. There is however one additional complication:
1.7660 +// blocks that have been allocated since the final checkpoint and
1.7661 +// mark, will not have been marked and so would be treated as
1.7662 +// unreachable and swept up. To prevent this, the allocator marks
1.7663 +// the bit map when allocating during the sweep phase. This leads,
1.7664 +// however, to a further complication -- objects may have been allocated
1.7665 +// but not yet initialized -- in the sense that the header isn't yet
1.7666 +// installed. The sweeper can not then determine the size of the block
1.7667 +// in order to skip over it. To deal with this case, we use a technique
1.7668 +// (due to Printezis) to encode such uninitialized block sizes in the
1.7669 +// bit map. Since the bit map uses a bit per every HeapWord, but the
1.7670 +// CMS generation has a minimum object size of 3 HeapWords, it follows
1.7671 +// that "normal marks" won't be adjacent in the bit map (there will
1.7672 +// always be at least two 0 bits between successive 1 bits). We make use
1.7673 +// of these "unused" bits to represent uninitialized blocks -- the bit
1.7674 +// corresponding to the start of the uninitialized object and the next
1.7675 +// bit are both set. Finally, a 1 bit marks the end of the object that
1.7676 +// started with the two consecutive 1 bits to indicate its potentially
1.7677 +// uninitialized state.
1.7678 +
1.7679 +size_t SweepClosure::do_blk_careful(HeapWord* addr) {
1.7680 + FreeChunk* fc = (FreeChunk*)addr;
1.7681 + size_t res;
1.7682 +
1.7683 + // check if we are done sweepinrg
1.7684 + if (addr == _limit) { // we have swept up to the limit, do nothing more
1.7685 + assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
1.7686 + "sweep _limit out of bounds");
1.7687 + // help the closure application finish
1.7688 + return pointer_delta(_sp->end(), _limit);
1.7689 + }
1.7690 + assert(addr <= _limit, "sweep invariant");
1.7691 +
1.7692 + // check if we should yield
1.7693 + do_yield_check(addr);
1.7694 + if (fc->isFree()) {
1.7695 + // Chunk that is already free
1.7696 + res = fc->size();
1.7697 + doAlreadyFreeChunk(fc);
1.7698 + debug_only(_sp->verifyFreeLists());
1.7699 + assert(res == fc->size(), "Don't expect the size to change");
1.7700 + NOT_PRODUCT(
1.7701 + _numObjectsAlreadyFree++;
1.7702 + _numWordsAlreadyFree += res;
1.7703 + )
1.7704 + NOT_PRODUCT(_last_fc = fc;)
1.7705 + } else if (!_bitMap->isMarked(addr)) {
1.7706 + // Chunk is fresh garbage
1.7707 + res = doGarbageChunk(fc);
1.7708 + debug_only(_sp->verifyFreeLists());
1.7709 + NOT_PRODUCT(
1.7710 + _numObjectsFreed++;
1.7711 + _numWordsFreed += res;
1.7712 + )
1.7713 + } else {
1.7714 + // Chunk that is alive.
1.7715 + res = doLiveChunk(fc);
1.7716 + debug_only(_sp->verifyFreeLists());
1.7717 + NOT_PRODUCT(
1.7718 + _numObjectsLive++;
1.7719 + _numWordsLive += res;
1.7720 + )
1.7721 + }
1.7722 + return res;
1.7723 +}
1.7724 +
1.7725 +// For the smart allocation, record following
1.7726 +// split deaths - a free chunk is removed from its free list because
1.7727 +// it is being split into two or more chunks.
1.7728 +// split birth - a free chunk is being added to its free list because
1.7729 +// a larger free chunk has been split and resulted in this free chunk.
1.7730 +// coal death - a free chunk is being removed from its free list because
1.7731 +// it is being coalesced into a large free chunk.
1.7732 +// coal birth - a free chunk is being added to its free list because
1.7733 +// it was created when two or more free chunks where coalesced into
1.7734 +// this free chunk.
1.7735 +//
1.7736 +// These statistics are used to determine the desired number of free
1.7737 +// chunks of a given size. The desired number is chosen to be relative
1.7738 +// to the end of a CMS sweep. The desired number at the end of a sweep
1.7739 +// is the
1.7740 +// count-at-end-of-previous-sweep (an amount that was enough)
1.7741 +// - count-at-beginning-of-current-sweep (the excess)
1.7742 +// + split-births (gains in this size during interval)
1.7743 +// - split-deaths (demands on this size during interval)
1.7744 +// where the interval is from the end of one sweep to the end of the
1.7745 +// next.
1.7746 +//
1.7747 +// When sweeping the sweeper maintains an accumulated chunk which is
1.7748 +// the chunk that is made up of chunks that have been coalesced. That
1.7749 +// will be termed the left-hand chunk. A new chunk of garbage that
1.7750 +// is being considered for coalescing will be referred to as the
1.7751 +// right-hand chunk.
1.7752 +//
1.7753 +// When making a decision on whether to coalesce a right-hand chunk with
1.7754 +// the current left-hand chunk, the current count vs. the desired count
1.7755 +// of the left-hand chunk is considered. Also if the right-hand chunk
1.7756 +// is near the large chunk at the end of the heap (see
1.7757 +// ConcurrentMarkSweepGeneration::isNearLargestChunk()), then the
1.7758 +// left-hand chunk is coalesced.
1.7759 +//
1.7760 +// When making a decision about whether to split a chunk, the desired count
1.7761 +// vs. the current count of the candidate to be split is also considered.
1.7762 +// If the candidate is underpopulated (currently fewer chunks than desired)
1.7763 +// a chunk of an overpopulated (currently more chunks than desired) size may
1.7764 +// be chosen. The "hint" associated with a free list, if non-null, points
1.7765 +// to a free list which may be overpopulated.
1.7766 +//
1.7767 +
1.7768 +void SweepClosure::doAlreadyFreeChunk(FreeChunk* fc) {
1.7769 + size_t size = fc->size();
1.7770 + // Chunks that cannot be coalesced are not in the
1.7771 + // free lists.
1.7772 + if (CMSTestInFreeList && !fc->cantCoalesce()) {
1.7773 + assert(_sp->verifyChunkInFreeLists(fc),
1.7774 + "free chunk should be in free lists");
1.7775 + }
1.7776 + // a chunk that is already free, should not have been
1.7777 + // marked in the bit map
1.7778 + HeapWord* addr = (HeapWord*) fc;
1.7779 + assert(!_bitMap->isMarked(addr), "free chunk should be unmarked");
1.7780 + // Verify that the bit map has no bits marked between
1.7781 + // addr and purported end of this block.
1.7782 + _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
1.7783 +
1.7784 + // Some chunks cannot be coalesced in under any circumstances.
1.7785 + // See the definition of cantCoalesce().
1.7786 + if (!fc->cantCoalesce()) {
1.7787 + // This chunk can potentially be coalesced.
1.7788 + if (_sp->adaptive_freelists()) {
1.7789 + // All the work is done in
1.7790 + doPostIsFreeOrGarbageChunk(fc, size);
1.7791 + } else { // Not adaptive free lists
1.7792 + // this is a free chunk that can potentially be coalesced by the sweeper;
1.7793 + if (!inFreeRange()) {
1.7794 + // if the next chunk is a free block that can't be coalesced
1.7795 + // it doesn't make sense to remove this chunk from the free lists
1.7796 + FreeChunk* nextChunk = (FreeChunk*)(addr + size);
1.7797 + assert((HeapWord*)nextChunk <= _limit, "sweep invariant");
1.7798 + if ((HeapWord*)nextChunk < _limit && // there's a next chunk...
1.7799 + nextChunk->isFree() && // which is free...
1.7800 + nextChunk->cantCoalesce()) { // ... but cant be coalesced
1.7801 + // nothing to do
1.7802 + } else {
1.7803 + // Potentially the start of a new free range:
1.7804 + // Don't eagerly remove it from the free lists.
1.7805 + // No need to remove it if it will just be put
1.7806 + // back again. (Also from a pragmatic point of view
1.7807 + // if it is a free block in a region that is beyond
1.7808 + // any allocated blocks, an assertion will fail)
1.7809 + // Remember the start of a free run.
1.7810 + initialize_free_range(addr, true);
1.7811 + // end - can coalesce with next chunk
1.7812 + }
1.7813 + } else {
1.7814 + // the midst of a free range, we are coalescing
1.7815 + debug_only(record_free_block_coalesced(fc);)
1.7816 + if (CMSTraceSweeper) {
1.7817 + gclog_or_tty->print(" -- pick up free block 0x%x (%d)\n", fc, size);
1.7818 + }
1.7819 + // remove it from the free lists
1.7820 + _sp->removeFreeChunkFromFreeLists(fc);
1.7821 + set_lastFreeRangeCoalesced(true);
1.7822 + // If the chunk is being coalesced and the current free range is
1.7823 + // in the free lists, remove the current free range so that it
1.7824 + // will be returned to the free lists in its entirety - all
1.7825 + // the coalesced pieces included.
1.7826 + if (freeRangeInFreeLists()) {
1.7827 + FreeChunk* ffc = (FreeChunk*) freeFinger();
1.7828 + assert(ffc->size() == pointer_delta(addr, freeFinger()),
1.7829 + "Size of free range is inconsistent with chunk size.");
1.7830 + if (CMSTestInFreeList) {
1.7831 + assert(_sp->verifyChunkInFreeLists(ffc),
1.7832 + "free range is not in free lists");
1.7833 + }
1.7834 + _sp->removeFreeChunkFromFreeLists(ffc);
1.7835 + set_freeRangeInFreeLists(false);
1.7836 + }
1.7837 + }
1.7838 + }
1.7839 + } else {
1.7840 + // Code path common to both original and adaptive free lists.
1.7841 +
1.7842 + // cant coalesce with previous block; this should be treated
1.7843 + // as the end of a free run if any
1.7844 + if (inFreeRange()) {
1.7845 + // we kicked some butt; time to pick up the garbage
1.7846 + assert(freeFinger() < addr, "the finger pointeth off base");
1.7847 + flushCurFreeChunk(freeFinger(), pointer_delta(addr, freeFinger()));
1.7848 + }
1.7849 + // else, nothing to do, just continue
1.7850 + }
1.7851 +}
1.7852 +
1.7853 +size_t SweepClosure::doGarbageChunk(FreeChunk* fc) {
1.7854 + // This is a chunk of garbage. It is not in any free list.
1.7855 + // Add it to a free list or let it possibly be coalesced into
1.7856 + // a larger chunk.
1.7857 + HeapWord* addr = (HeapWord*) fc;
1.7858 + size_t size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
1.7859 +
1.7860 + if (_sp->adaptive_freelists()) {
1.7861 + // Verify that the bit map has no bits marked between
1.7862 + // addr and purported end of just dead object.
1.7863 + _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
1.7864 +
1.7865 + doPostIsFreeOrGarbageChunk(fc, size);
1.7866 + } else {
1.7867 + if (!inFreeRange()) {
1.7868 + // start of a new free range
1.7869 + assert(size > 0, "A free range should have a size");
1.7870 + initialize_free_range(addr, false);
1.7871 +
1.7872 + } else {
1.7873 + // this will be swept up when we hit the end of the
1.7874 + // free range
1.7875 + if (CMSTraceSweeper) {
1.7876 + gclog_or_tty->print(" -- pick up garbage 0x%x (%d) \n", fc, size);
1.7877 + }
1.7878 + // If the chunk is being coalesced and the current free range is
1.7879 + // in the free lists, remove the current free range so that it
1.7880 + // will be returned to the free lists in its entirety - all
1.7881 + // the coalesced pieces included.
1.7882 + if (freeRangeInFreeLists()) {
1.7883 + FreeChunk* ffc = (FreeChunk*)freeFinger();
1.7884 + assert(ffc->size() == pointer_delta(addr, freeFinger()),
1.7885 + "Size of free range is inconsistent with chunk size.");
1.7886 + if (CMSTestInFreeList) {
1.7887 + assert(_sp->verifyChunkInFreeLists(ffc),
1.7888 + "free range is not in free lists");
1.7889 + }
1.7890 + _sp->removeFreeChunkFromFreeLists(ffc);
1.7891 + set_freeRangeInFreeLists(false);
1.7892 + }
1.7893 + set_lastFreeRangeCoalesced(true);
1.7894 + }
1.7895 + // this will be swept up when we hit the end of the free range
1.7896 +
1.7897 + // Verify that the bit map has no bits marked between
1.7898 + // addr and purported end of just dead object.
1.7899 + _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
1.7900 + }
1.7901 + return size;
1.7902 +}
1.7903 +
1.7904 +size_t SweepClosure::doLiveChunk(FreeChunk* fc) {
1.7905 + HeapWord* addr = (HeapWord*) fc;
1.7906 + // The sweeper has just found a live object. Return any accumulated
1.7907 + // left hand chunk to the free lists.
1.7908 + if (inFreeRange()) {
1.7909 + if (_sp->adaptive_freelists()) {
1.7910 + flushCurFreeChunk(freeFinger(),
1.7911 + pointer_delta(addr, freeFinger()));
1.7912 + } else { // not adaptive freelists
1.7913 + set_inFreeRange(false);
1.7914 + // Add the free range back to the free list if it is not already
1.7915 + // there.
1.7916 + if (!freeRangeInFreeLists()) {
1.7917 + assert(freeFinger() < addr, "the finger pointeth off base");
1.7918 + if (CMSTraceSweeper) {
1.7919 + gclog_or_tty->print("Sweep:put_free_blk 0x%x (%d) "
1.7920 + "[coalesced:%d]\n",
1.7921 + freeFinger(), pointer_delta(addr, freeFinger()),
1.7922 + lastFreeRangeCoalesced());
1.7923 + }
1.7924 + _sp->addChunkAndRepairOffsetTable(freeFinger(),
1.7925 + pointer_delta(addr, freeFinger()), lastFreeRangeCoalesced());
1.7926 + }
1.7927 + }
1.7928 + }
1.7929 +
1.7930 + // Common code path for original and adaptive free lists.
1.7931 +
1.7932 + // this object is live: we'd normally expect this to be
1.7933 + // an oop, and like to assert the following:
1.7934 + // assert(oop(addr)->is_oop(), "live block should be an oop");
1.7935 + // However, as we commented above, this may be an object whose
1.7936 + // header hasn't yet been initialized.
1.7937 + size_t size;
1.7938 + assert(_bitMap->isMarked(addr), "Tautology for this control point");
1.7939 + if (_bitMap->isMarked(addr + 1)) {
1.7940 + // Determine the size from the bit map, rather than trying to
1.7941 + // compute it from the object header.
1.7942 + HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
1.7943 + size = pointer_delta(nextOneAddr + 1, addr);
1.7944 + assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
1.7945 + "alignment problem");
1.7946 +
1.7947 + #ifdef DEBUG
1.7948 + if (oop(addr)->klass() != NULL &&
1.7949 + ( !_collector->cms_should_unload_classes()
1.7950 + || oop(addr)->is_parsable())) {
1.7951 + // Ignore mark word because we are running concurrent with mutators
1.7952 + assert(oop(addr)->is_oop(true), "live block should be an oop");
1.7953 + assert(size ==
1.7954 + CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()),
1.7955 + "P-mark and computed size do not agree");
1.7956 + }
1.7957 + #endif
1.7958 +
1.7959 + } else {
1.7960 + // This should be an initialized object that's alive.
1.7961 + assert(oop(addr)->klass() != NULL &&
1.7962 + (!_collector->cms_should_unload_classes()
1.7963 + || oop(addr)->is_parsable()),
1.7964 + "Should be an initialized object");
1.7965 + // Ignore mark word because we are running concurrent with mutators
1.7966 + assert(oop(addr)->is_oop(true), "live block should be an oop");
1.7967 + // Verify that the bit map has no bits marked between
1.7968 + // addr and purported end of this block.
1.7969 + size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
1.7970 + assert(size >= 3, "Necessary for Printezis marks to work");
1.7971 + assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
1.7972 + DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
1.7973 + }
1.7974 + return size;
1.7975 +}
1.7976 +
1.7977 +void SweepClosure::doPostIsFreeOrGarbageChunk(FreeChunk* fc,
1.7978 + size_t chunkSize) {
1.7979 + // doPostIsFreeOrGarbageChunk() should only be called in the smart allocation
1.7980 + // scheme.
1.7981 + bool fcInFreeLists = fc->isFree();
1.7982 + assert(_sp->adaptive_freelists(), "Should only be used in this case.");
1.7983 + assert((HeapWord*)fc <= _limit, "sweep invariant");
1.7984 + if (CMSTestInFreeList && fcInFreeLists) {
1.7985 + assert(_sp->verifyChunkInFreeLists(fc),
1.7986 + "free chunk is not in free lists");
1.7987 + }
1.7988 +
1.7989 +
1.7990 + if (CMSTraceSweeper) {
1.7991 + gclog_or_tty->print_cr(" -- pick up another chunk at 0x%x (%d)", fc, chunkSize);
1.7992 + }
1.7993 +
1.7994 + HeapWord* addr = (HeapWord*) fc;
1.7995 +
1.7996 + bool coalesce;
1.7997 + size_t left = pointer_delta(addr, freeFinger());
1.7998 + size_t right = chunkSize;
1.7999 + switch (FLSCoalescePolicy) {
1.8000 + // numeric value forms a coalition aggressiveness metric
1.8001 + case 0: { // never coalesce
1.8002 + coalesce = false;
1.8003 + break;
1.8004 + }
1.8005 + case 1: { // coalesce if left & right chunks on overpopulated lists
1.8006 + coalesce = _sp->coalOverPopulated(left) &&
1.8007 + _sp->coalOverPopulated(right);
1.8008 + break;
1.8009 + }
1.8010 + case 2: { // coalesce if left chunk on overpopulated list (default)
1.8011 + coalesce = _sp->coalOverPopulated(left);
1.8012 + break;
1.8013 + }
1.8014 + case 3: { // coalesce if left OR right chunk on overpopulated list
1.8015 + coalesce = _sp->coalOverPopulated(left) ||
1.8016 + _sp->coalOverPopulated(right);
1.8017 + break;
1.8018 + }
1.8019 + case 4: { // always coalesce
1.8020 + coalesce = true;
1.8021 + break;
1.8022 + }
1.8023 + default:
1.8024 + ShouldNotReachHere();
1.8025 + }
1.8026 +
1.8027 + // Should the current free range be coalesced?
1.8028 + // If the chunk is in a free range and either we decided to coalesce above
1.8029 + // or the chunk is near the large block at the end of the heap
1.8030 + // (isNearLargestChunk() returns true), then coalesce this chunk.
1.8031 + bool doCoalesce = inFreeRange() &&
1.8032 + (coalesce || _g->isNearLargestChunk((HeapWord*)fc));
1.8033 + if (doCoalesce) {
1.8034 + // Coalesce the current free range on the left with the new
1.8035 + // chunk on the right. If either is on a free list,
1.8036 + // it must be removed from the list and stashed in the closure.
1.8037 + if (freeRangeInFreeLists()) {
1.8038 + FreeChunk* ffc = (FreeChunk*)freeFinger();
1.8039 + assert(ffc->size() == pointer_delta(addr, freeFinger()),
1.8040 + "Size of free range is inconsistent with chunk size.");
1.8041 + if (CMSTestInFreeList) {
1.8042 + assert(_sp->verifyChunkInFreeLists(ffc),
1.8043 + "Chunk is not in free lists");
1.8044 + }
1.8045 + _sp->coalDeath(ffc->size());
1.8046 + _sp->removeFreeChunkFromFreeLists(ffc);
1.8047 + set_freeRangeInFreeLists(false);
1.8048 + }
1.8049 + if (fcInFreeLists) {
1.8050 + _sp->coalDeath(chunkSize);
1.8051 + assert(fc->size() == chunkSize,
1.8052 + "The chunk has the wrong size or is not in the free lists");
1.8053 + _sp->removeFreeChunkFromFreeLists(fc);
1.8054 + }
1.8055 + set_lastFreeRangeCoalesced(true);
1.8056 + } else { // not in a free range and/or should not coalesce
1.8057 + // Return the current free range and start a new one.
1.8058 + if (inFreeRange()) {
1.8059 + // In a free range but cannot coalesce with the right hand chunk.
1.8060 + // Put the current free range into the free lists.
1.8061 + flushCurFreeChunk(freeFinger(),
1.8062 + pointer_delta(addr, freeFinger()));
1.8063 + }
1.8064 + // Set up for new free range. Pass along whether the right hand
1.8065 + // chunk is in the free lists.
1.8066 + initialize_free_range((HeapWord*)fc, fcInFreeLists);
1.8067 + }
1.8068 +}
1.8069 +void SweepClosure::flushCurFreeChunk(HeapWord* chunk, size_t size) {
1.8070 + assert(inFreeRange(), "Should only be called if currently in a free range.");
1.8071 + assert(size > 0,
1.8072 + "A zero sized chunk cannot be added to the free lists.");
1.8073 + if (!freeRangeInFreeLists()) {
1.8074 + if(CMSTestInFreeList) {
1.8075 + FreeChunk* fc = (FreeChunk*) chunk;
1.8076 + fc->setSize(size);
1.8077 + assert(!_sp->verifyChunkInFreeLists(fc),
1.8078 + "chunk should not be in free lists yet");
1.8079 + }
1.8080 + if (CMSTraceSweeper) {
1.8081 + gclog_or_tty->print_cr(" -- add free block 0x%x (%d) to free lists",
1.8082 + chunk, size);
1.8083 + }
1.8084 + // A new free range is going to be starting. The current
1.8085 + // free range has not been added to the free lists yet or
1.8086 + // was removed so add it back.
1.8087 + // If the current free range was coalesced, then the death
1.8088 + // of the free range was recorded. Record a birth now.
1.8089 + if (lastFreeRangeCoalesced()) {
1.8090 + _sp->coalBirth(size);
1.8091 + }
1.8092 + _sp->addChunkAndRepairOffsetTable(chunk, size,
1.8093 + lastFreeRangeCoalesced());
1.8094 + }
1.8095 + set_inFreeRange(false);
1.8096 + set_freeRangeInFreeLists(false);
1.8097 +}
1.8098 +
1.8099 +// We take a break if we've been at this for a while,
1.8100 +// so as to avoid monopolizing the locks involved.
1.8101 +void SweepClosure::do_yield_work(HeapWord* addr) {
1.8102 + // Return current free chunk being used for coalescing (if any)
1.8103 + // to the appropriate freelist. After yielding, the next
1.8104 + // free block encountered will start a coalescing range of
1.8105 + // free blocks. If the next free block is adjacent to the
1.8106 + // chunk just flushed, they will need to wait for the next
1.8107 + // sweep to be coalesced.
1.8108 + if (inFreeRange()) {
1.8109 + flushCurFreeChunk(freeFinger(), pointer_delta(addr, freeFinger()));
1.8110 + }
1.8111 +
1.8112 + // First give up the locks, then yield, then re-lock.
1.8113 + // We should probably use a constructor/destructor idiom to
1.8114 + // do this unlock/lock or modify the MutexUnlocker class to
1.8115 + // serve our purpose. XXX
1.8116 + assert_lock_strong(_bitMap->lock());
1.8117 + assert_lock_strong(_freelistLock);
1.8118 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.8119 + "CMS thread should hold CMS token");
1.8120 + _bitMap->lock()->unlock();
1.8121 + _freelistLock->unlock();
1.8122 + ConcurrentMarkSweepThread::desynchronize(true);
1.8123 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.8124 + _collector->stopTimer();
1.8125 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.8126 + if (PrintCMSStatistics != 0) {
1.8127 + _collector->incrementYields();
1.8128 + }
1.8129 + _collector->icms_wait();
1.8130 +
1.8131 + // See the comment in coordinator_yield()
1.8132 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.8133 + ConcurrentMarkSweepThread::should_yield() &&
1.8134 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.8135 + os::sleep(Thread::current(), 1, false);
1.8136 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.8137 + }
1.8138 +
1.8139 + ConcurrentMarkSweepThread::synchronize(true);
1.8140 + _freelistLock->lock();
1.8141 + _bitMap->lock()->lock_without_safepoint_check();
1.8142 + _collector->startTimer();
1.8143 +}
1.8144 +
1.8145 +#ifndef PRODUCT
1.8146 +// This is actually very useful in a product build if it can
1.8147 +// be called from the debugger. Compile it into the product
1.8148 +// as needed.
1.8149 +bool debug_verifyChunkInFreeLists(FreeChunk* fc) {
1.8150 + return debug_cms_space->verifyChunkInFreeLists(fc);
1.8151 +}
1.8152 +
1.8153 +void SweepClosure::record_free_block_coalesced(FreeChunk* fc) const {
1.8154 + if (CMSTraceSweeper) {
1.8155 + gclog_or_tty->print("Sweep:coal_free_blk 0x%x (%d)\n", fc, fc->size());
1.8156 + }
1.8157 +}
1.8158 +#endif
1.8159 +
1.8160 +// CMSIsAliveClosure
1.8161 +bool CMSIsAliveClosure::do_object_b(oop obj) {
1.8162 + HeapWord* addr = (HeapWord*)obj;
1.8163 + return addr != NULL &&
1.8164 + (!_span.contains(addr) || _bit_map->isMarked(addr));
1.8165 +}
1.8166 +
1.8167 +// CMSKeepAliveClosure: the serial version
1.8168 +void CMSKeepAliveClosure::do_oop(oop* p) {
1.8169 + oop this_oop = *p;
1.8170 + HeapWord* addr = (HeapWord*)this_oop;
1.8171 + if (_span.contains(addr) &&
1.8172 + !_bit_map->isMarked(addr)) {
1.8173 + _bit_map->mark(addr);
1.8174 + bool simulate_overflow = false;
1.8175 + NOT_PRODUCT(
1.8176 + if (CMSMarkStackOverflowALot &&
1.8177 + _collector->simulate_overflow()) {
1.8178 + // simulate a stack overflow
1.8179 + simulate_overflow = true;
1.8180 + }
1.8181 + )
1.8182 + if (simulate_overflow || !_mark_stack->push(this_oop)) {
1.8183 + _collector->push_on_overflow_list(this_oop);
1.8184 + _collector->_ser_kac_ovflw++;
1.8185 + }
1.8186 + }
1.8187 +}
1.8188 +
1.8189 +// CMSParKeepAliveClosure: a parallel version of the above.
1.8190 +// The work queues are private to each closure (thread),
1.8191 +// but (may be) available for stealing by other threads.
1.8192 +void CMSParKeepAliveClosure::do_oop(oop* p) {
1.8193 + oop this_oop = *p;
1.8194 + HeapWord* addr = (HeapWord*)this_oop;
1.8195 + if (_span.contains(addr) &&
1.8196 + !_bit_map->isMarked(addr)) {
1.8197 + // In general, during recursive tracing, several threads
1.8198 + // may be concurrently getting here; the first one to
1.8199 + // "tag" it, claims it.
1.8200 + if (_bit_map->par_mark(addr)) {
1.8201 + bool res = _work_queue->push(this_oop);
1.8202 + assert(res, "Low water mark should be much less than capacity");
1.8203 + // Do a recursive trim in the hope that this will keep
1.8204 + // stack usage lower, but leave some oops for potential stealers
1.8205 + trim_queue(_low_water_mark);
1.8206 + } // Else, another thread got there first
1.8207 + }
1.8208 +}
1.8209 +
1.8210 +void CMSParKeepAliveClosure::trim_queue(uint max) {
1.8211 + while (_work_queue->size() > max) {
1.8212 + oop new_oop;
1.8213 + if (_work_queue->pop_local(new_oop)) {
1.8214 + assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
1.8215 + assert(_bit_map->isMarked((HeapWord*)new_oop),
1.8216 + "no white objects on this stack!");
1.8217 + assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop");
1.8218 + // iterate over the oops in this oop, marking and pushing
1.8219 + // the ones in CMS heap (i.e. in _span).
1.8220 + new_oop->oop_iterate(&_mark_and_push);
1.8221 + }
1.8222 + }
1.8223 +}
1.8224 +
1.8225 +void CMSInnerParMarkAndPushClosure::do_oop(oop* p) {
1.8226 + oop this_oop = *p;
1.8227 + HeapWord* addr = (HeapWord*)this_oop;
1.8228 + if (_span.contains(addr) &&
1.8229 + !_bit_map->isMarked(addr)) {
1.8230 + if (_bit_map->par_mark(addr)) {
1.8231 + bool simulate_overflow = false;
1.8232 + NOT_PRODUCT(
1.8233 + if (CMSMarkStackOverflowALot &&
1.8234 + _collector->par_simulate_overflow()) {
1.8235 + // simulate a stack overflow
1.8236 + simulate_overflow = true;
1.8237 + }
1.8238 + )
1.8239 + if (simulate_overflow || !_work_queue->push(this_oop)) {
1.8240 + _collector->par_push_on_overflow_list(this_oop);
1.8241 + _collector->_par_kac_ovflw++;
1.8242 + }
1.8243 + } // Else another thread got there already
1.8244 + }
1.8245 +}
1.8246 +
1.8247 +//////////////////////////////////////////////////////////////////
1.8248 +// CMSExpansionCause /////////////////////////////
1.8249 +//////////////////////////////////////////////////////////////////
1.8250 +const char* CMSExpansionCause::to_string(CMSExpansionCause::Cause cause) {
1.8251 + switch (cause) {
1.8252 + case _no_expansion:
1.8253 + return "No expansion";
1.8254 + case _satisfy_free_ratio:
1.8255 + return "Free ratio";
1.8256 + case _satisfy_promotion:
1.8257 + return "Satisfy promotion";
1.8258 + case _satisfy_allocation:
1.8259 + return "allocation";
1.8260 + case _allocate_par_lab:
1.8261 + return "Par LAB";
1.8262 + case _allocate_par_spooling_space:
1.8263 + return "Par Spooling Space";
1.8264 + case _adaptive_size_policy:
1.8265 + return "Ergonomics";
1.8266 + default:
1.8267 + return "unknown";
1.8268 + }
1.8269 +}
1.8270 +
1.8271 +void CMSDrainMarkingStackClosure::do_void() {
1.8272 + // the max number to take from overflow list at a time
1.8273 + const size_t num = _mark_stack->capacity()/4;
1.8274 + while (!_mark_stack->isEmpty() ||
1.8275 + // if stack is empty, check the overflow list
1.8276 + _collector->take_from_overflow_list(num, _mark_stack)) {
1.8277 + oop this_oop = _mark_stack->pop();
1.8278 + HeapWord* addr = (HeapWord*)this_oop;
1.8279 + assert(_span.contains(addr), "Should be within span");
1.8280 + assert(_bit_map->isMarked(addr), "Should be marked");
1.8281 + assert(this_oop->is_oop(), "Should be an oop");
1.8282 + this_oop->oop_iterate(_keep_alive);
1.8283 + }
1.8284 +}
1.8285 +
1.8286 +void CMSParDrainMarkingStackClosure::do_void() {
1.8287 + // drain queue
1.8288 + trim_queue(0);
1.8289 +}
1.8290 +
1.8291 +// Trim our work_queue so its length is below max at return
1.8292 +void CMSParDrainMarkingStackClosure::trim_queue(uint max) {
1.8293 + while (_work_queue->size() > max) {
1.8294 + oop new_oop;
1.8295 + if (_work_queue->pop_local(new_oop)) {
1.8296 + assert(new_oop->is_oop(), "Expected an oop");
1.8297 + assert(_bit_map->isMarked((HeapWord*)new_oop),
1.8298 + "no white objects on this stack!");
1.8299 + assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop");
1.8300 + // iterate over the oops in this oop, marking and pushing
1.8301 + // the ones in CMS heap (i.e. in _span).
1.8302 + new_oop->oop_iterate(&_mark_and_push);
1.8303 + }
1.8304 + }
1.8305 +}
1.8306 +
1.8307 +////////////////////////////////////////////////////////////////////
1.8308 +// Support for Marking Stack Overflow list handling and related code
1.8309 +////////////////////////////////////////////////////////////////////
1.8310 +// Much of the following code is similar in shape and spirit to the
1.8311 +// code used in ParNewGC. We should try and share that code
1.8312 +// as much as possible in the future.
1.8313 +
1.8314 +#ifndef PRODUCT
1.8315 +// Debugging support for CMSStackOverflowALot
1.8316 +
1.8317 +// It's OK to call this multi-threaded; the worst thing
1.8318 +// that can happen is that we'll get a bunch of closely
1.8319 +// spaced simulated oveflows, but that's OK, in fact
1.8320 +// probably good as it would exercise the overflow code
1.8321 +// under contention.
1.8322 +bool CMSCollector::simulate_overflow() {
1.8323 + if (_overflow_counter-- <= 0) { // just being defensive
1.8324 + _overflow_counter = CMSMarkStackOverflowInterval;
1.8325 + return true;
1.8326 + } else {
1.8327 + return false;
1.8328 + }
1.8329 +}
1.8330 +
1.8331 +bool CMSCollector::par_simulate_overflow() {
1.8332 + return simulate_overflow();
1.8333 +}
1.8334 +#endif
1.8335 +
1.8336 +// Single-threaded
1.8337 +bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) {
1.8338 + assert(stack->isEmpty(), "Expected precondition");
1.8339 + assert(stack->capacity() > num, "Shouldn't bite more than can chew");
1.8340 + size_t i = num;
1.8341 + oop cur = _overflow_list;
1.8342 + const markOop proto = markOopDesc::prototype();
1.8343 + NOT_PRODUCT(size_t n = 0;)
1.8344 + for (oop next; i > 0 && cur != NULL; cur = next, i--) {
1.8345 + next = oop(cur->mark());
1.8346 + cur->set_mark(proto); // until proven otherwise
1.8347 + assert(cur->is_oop(), "Should be an oop");
1.8348 + bool res = stack->push(cur);
1.8349 + assert(res, "Bit off more than can chew?");
1.8350 + NOT_PRODUCT(n++;)
1.8351 + }
1.8352 + _overflow_list = cur;
1.8353 +#ifndef PRODUCT
1.8354 + assert(_num_par_pushes >= n, "Too many pops?");
1.8355 + _num_par_pushes -=n;
1.8356 +#endif
1.8357 + return !stack->isEmpty();
1.8358 +}
1.8359 +
1.8360 +// Multi-threaded; use CAS to break off a prefix
1.8361 +bool CMSCollector::par_take_from_overflow_list(size_t num,
1.8362 + OopTaskQueue* work_q) {
1.8363 + assert(work_q->size() == 0, "That's the current policy");
1.8364 + assert(num < work_q->max_elems(), "Can't bite more than we can chew");
1.8365 + if (_overflow_list == NULL) {
1.8366 + return false;
1.8367 + }
1.8368 + // Grab the entire list; we'll put back a suffix
1.8369 + oop prefix = (oop)Atomic::xchg_ptr(NULL, &_overflow_list);
1.8370 + if (prefix == NULL) { // someone grabbed it before we did ...
1.8371 + // ... we could spin for a short while, but for now we don't
1.8372 + return false;
1.8373 + }
1.8374 + size_t i = num;
1.8375 + oop cur = prefix;
1.8376 + for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--);
1.8377 + if (cur->mark() != NULL) {
1.8378 + oop suffix_head = cur->mark(); // suffix will be put back on global list
1.8379 + cur->set_mark(NULL); // break off suffix
1.8380 + // Find tail of suffix so we can prepend suffix to global list
1.8381 + for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark()));
1.8382 + oop suffix_tail = cur;
1.8383 + assert(suffix_tail != NULL && suffix_tail->mark() == NULL,
1.8384 + "Tautology");
1.8385 + oop observed_overflow_list = _overflow_list;
1.8386 + do {
1.8387 + cur = observed_overflow_list;
1.8388 + suffix_tail->set_mark(markOop(cur));
1.8389 + observed_overflow_list =
1.8390 + (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur);
1.8391 + } while (cur != observed_overflow_list);
1.8392 + }
1.8393 +
1.8394 + // Push the prefix elements on work_q
1.8395 + assert(prefix != NULL, "control point invariant");
1.8396 + const markOop proto = markOopDesc::prototype();
1.8397 + oop next;
1.8398 + NOT_PRODUCT(size_t n = 0;)
1.8399 + for (cur = prefix; cur != NULL; cur = next) {
1.8400 + next = oop(cur->mark());
1.8401 + cur->set_mark(proto); // until proven otherwise
1.8402 + assert(cur->is_oop(), "Should be an oop");
1.8403 + bool res = work_q->push(cur);
1.8404 + assert(res, "Bit off more than we can chew?");
1.8405 + NOT_PRODUCT(n++;)
1.8406 + }
1.8407 +#ifndef PRODUCT
1.8408 + assert(_num_par_pushes >= n, "Too many pops?");
1.8409 + Atomic::add_ptr(-(intptr_t)n, &_num_par_pushes);
1.8410 +#endif
1.8411 + return true;
1.8412 +}
1.8413 +
1.8414 +// Single-threaded
1.8415 +void CMSCollector::push_on_overflow_list(oop p) {
1.8416 + NOT_PRODUCT(_num_par_pushes++;)
1.8417 + assert(p->is_oop(), "Not an oop");
1.8418 + preserve_mark_if_necessary(p);
1.8419 + p->set_mark((markOop)_overflow_list);
1.8420 + _overflow_list = p;
1.8421 +}
1.8422 +
1.8423 +// Multi-threaded; use CAS to prepend to overflow list
1.8424 +void CMSCollector::par_push_on_overflow_list(oop p) {
1.8425 + NOT_PRODUCT(Atomic::inc_ptr(&_num_par_pushes);)
1.8426 + assert(p->is_oop(), "Not an oop");
1.8427 + par_preserve_mark_if_necessary(p);
1.8428 + oop observed_overflow_list = _overflow_list;
1.8429 + oop cur_overflow_list;
1.8430 + do {
1.8431 + cur_overflow_list = observed_overflow_list;
1.8432 + p->set_mark(markOop(cur_overflow_list));
1.8433 + observed_overflow_list =
1.8434 + (oop) Atomic::cmpxchg_ptr(p, &_overflow_list, cur_overflow_list);
1.8435 + } while (cur_overflow_list != observed_overflow_list);
1.8436 +}
1.8437 +
1.8438 +// Single threaded
1.8439 +// General Note on GrowableArray: pushes may silently fail
1.8440 +// because we are (temporarily) out of C-heap for expanding
1.8441 +// the stack. The problem is quite ubiquitous and affects
1.8442 +// a lot of code in the JVM. The prudent thing for GrowableArray
1.8443 +// to do (for now) is to exit with an error. However, that may
1.8444 +// be too draconian in some cases because the caller may be
1.8445 +// able to recover without much harm. For suych cases, we
1.8446 +// should probably introduce a "soft_push" method which returns
1.8447 +// an indication of success or failure with the assumption that
1.8448 +// the caller may be able to recover from a failure; code in
1.8449 +// the VM can then be changed, incrementally, to deal with such
1.8450 +// failures where possible, thus, incrementally hardening the VM
1.8451 +// in such low resource situations.
1.8452 +void CMSCollector::preserve_mark_work(oop p, markOop m) {
1.8453 + int PreserveMarkStackSize = 128;
1.8454 +
1.8455 + if (_preserved_oop_stack == NULL) {
1.8456 + assert(_preserved_mark_stack == NULL,
1.8457 + "bijection with preserved_oop_stack");
1.8458 + // Allocate the stacks
1.8459 + _preserved_oop_stack = new (ResourceObj::C_HEAP)
1.8460 + GrowableArray<oop>(PreserveMarkStackSize, true);
1.8461 + _preserved_mark_stack = new (ResourceObj::C_HEAP)
1.8462 + GrowableArray<markOop>(PreserveMarkStackSize, true);
1.8463 + if (_preserved_oop_stack == NULL || _preserved_mark_stack == NULL) {
1.8464 + vm_exit_out_of_memory(2* PreserveMarkStackSize * sizeof(oop) /* punt */,
1.8465 + "Preserved Mark/Oop Stack for CMS (C-heap)");
1.8466 + }
1.8467 + }
1.8468 + _preserved_oop_stack->push(p);
1.8469 + _preserved_mark_stack->push(m);
1.8470 + assert(m == p->mark(), "Mark word changed");
1.8471 + assert(_preserved_oop_stack->length() == _preserved_mark_stack->length(),
1.8472 + "bijection");
1.8473 +}
1.8474 +
1.8475 +// Single threaded
1.8476 +void CMSCollector::preserve_mark_if_necessary(oop p) {
1.8477 + markOop m = p->mark();
1.8478 + if (m->must_be_preserved(p)) {
1.8479 + preserve_mark_work(p, m);
1.8480 + }
1.8481 +}
1.8482 +
1.8483 +void CMSCollector::par_preserve_mark_if_necessary(oop p) {
1.8484 + markOop m = p->mark();
1.8485 + if (m->must_be_preserved(p)) {
1.8486 + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1.8487 + // Even though we read the mark word without holding
1.8488 + // the lock, we are assured that it will not change
1.8489 + // because we "own" this oop, so no other thread can
1.8490 + // be trying to push it on the overflow list; see
1.8491 + // the assertion in preserve_mark_work() that checks
1.8492 + // that m == p->mark().
1.8493 + preserve_mark_work(p, m);
1.8494 + }
1.8495 +}
1.8496 +
1.8497 +// We should be able to do this multi-threaded,
1.8498 +// a chunk of stack being a task (this is
1.8499 +// correct because each oop only ever appears
1.8500 +// once in the overflow list. However, it's
1.8501 +// not very easy to completely overlap this with
1.8502 +// other operations, so will generally not be done
1.8503 +// until all work's been completed. Because we
1.8504 +// expect the preserved oop stack (set) to be small,
1.8505 +// it's probably fine to do this single-threaded.
1.8506 +// We can explore cleverer concurrent/overlapped/parallel
1.8507 +// processing of preserved marks if we feel the
1.8508 +// need for this in the future. Stack overflow should
1.8509 +// be so rare in practice and, when it happens, its
1.8510 +// effect on performance so great that this will
1.8511 +// likely just be in the noise anyway.
1.8512 +void CMSCollector::restore_preserved_marks_if_any() {
1.8513 + if (_preserved_oop_stack == NULL) {
1.8514 + assert(_preserved_mark_stack == NULL,
1.8515 + "bijection with preserved_oop_stack");
1.8516 + return;
1.8517 + }
1.8518 +
1.8519 + assert(SafepointSynchronize::is_at_safepoint(),
1.8520 + "world should be stopped");
1.8521 + assert(Thread::current()->is_ConcurrentGC_thread() ||
1.8522 + Thread::current()->is_VM_thread(),
1.8523 + "should be single-threaded");
1.8524 +
1.8525 + int length = _preserved_oop_stack->length();
1.8526 + assert(_preserved_mark_stack->length() == length, "bijection");
1.8527 + for (int i = 0; i < length; i++) {
1.8528 + oop p = _preserved_oop_stack->at(i);
1.8529 + assert(p->is_oop(), "Should be an oop");
1.8530 + assert(_span.contains(p), "oop should be in _span");
1.8531 + assert(p->mark() == markOopDesc::prototype(),
1.8532 + "Set when taken from overflow list");
1.8533 + markOop m = _preserved_mark_stack->at(i);
1.8534 + p->set_mark(m);
1.8535 + }
1.8536 + _preserved_mark_stack->clear();
1.8537 + _preserved_oop_stack->clear();
1.8538 + assert(_preserved_mark_stack->is_empty() &&
1.8539 + _preserved_oop_stack->is_empty(),
1.8540 + "stacks were cleared above");
1.8541 +}
1.8542 +
1.8543 +#ifndef PRODUCT
1.8544 +bool CMSCollector::no_preserved_marks() const {
1.8545 + return ( ( _preserved_mark_stack == NULL
1.8546 + && _preserved_oop_stack == NULL)
1.8547 + || ( _preserved_mark_stack->is_empty()
1.8548 + && _preserved_oop_stack->is_empty()));
1.8549 +}
1.8550 +#endif
1.8551 +
1.8552 +CMSAdaptiveSizePolicy* ASConcurrentMarkSweepGeneration::cms_size_policy() const
1.8553 +{
1.8554 + GenCollectedHeap* gch = (GenCollectedHeap*) GenCollectedHeap::heap();
1.8555 + CMSAdaptiveSizePolicy* size_policy =
1.8556 + (CMSAdaptiveSizePolicy*) gch->gen_policy()->size_policy();
1.8557 + assert(size_policy->is_gc_cms_adaptive_size_policy(),
1.8558 + "Wrong type for size policy");
1.8559 + return size_policy;
1.8560 +}
1.8561 +
1.8562 +void ASConcurrentMarkSweepGeneration::resize(size_t cur_promo_size,
1.8563 + size_t desired_promo_size) {
1.8564 + if (cur_promo_size < desired_promo_size) {
1.8565 + size_t expand_bytes = desired_promo_size - cur_promo_size;
1.8566 + if (PrintAdaptiveSizePolicy && Verbose) {
1.8567 + gclog_or_tty->print_cr(" ASConcurrentMarkSweepGeneration::resize "
1.8568 + "Expanding tenured generation by " SIZE_FORMAT " (bytes)",
1.8569 + expand_bytes);
1.8570 + }
1.8571 + expand(expand_bytes,
1.8572 + MinHeapDeltaBytes,
1.8573 + CMSExpansionCause::_adaptive_size_policy);
1.8574 + } else if (desired_promo_size < cur_promo_size) {
1.8575 + size_t shrink_bytes = cur_promo_size - desired_promo_size;
1.8576 + if (PrintAdaptiveSizePolicy && Verbose) {
1.8577 + gclog_or_tty->print_cr(" ASConcurrentMarkSweepGeneration::resize "
1.8578 + "Shrinking tenured generation by " SIZE_FORMAT " (bytes)",
1.8579 + shrink_bytes);
1.8580 + }
1.8581 + shrink(shrink_bytes);
1.8582 + }
1.8583 +}
1.8584 +
1.8585 +CMSGCAdaptivePolicyCounters* ASConcurrentMarkSweepGeneration::gc_adaptive_policy_counters() {
1.8586 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.8587 + CMSGCAdaptivePolicyCounters* counters =
1.8588 + (CMSGCAdaptivePolicyCounters*) gch->collector_policy()->counters();
1.8589 + assert(counters->kind() == GCPolicyCounters::CMSGCAdaptivePolicyCountersKind,
1.8590 + "Wrong kind of counters");
1.8591 + return counters;
1.8592 +}
1.8593 +
1.8594 +
1.8595 +void ASConcurrentMarkSweepGeneration::update_counters() {
1.8596 + if (UsePerfData) {
1.8597 + _space_counters->update_all();
1.8598 + _gen_counters->update_all();
1.8599 + CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
1.8600 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.8601 + CMSGCStats* gc_stats_l = (CMSGCStats*) gc_stats();
1.8602 + assert(gc_stats_l->kind() == GCStats::CMSGCStatsKind,
1.8603 + "Wrong gc statistics type");
1.8604 + counters->update_counters(gc_stats_l);
1.8605 + }
1.8606 +}
1.8607 +
1.8608 +void ASConcurrentMarkSweepGeneration::update_counters(size_t used) {
1.8609 + if (UsePerfData) {
1.8610 + _space_counters->update_used(used);
1.8611 + _space_counters->update_capacity();
1.8612 + _gen_counters->update_all();
1.8613 +
1.8614 + CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
1.8615 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.8616 + CMSGCStats* gc_stats_l = (CMSGCStats*) gc_stats();
1.8617 + assert(gc_stats_l->kind() == GCStats::CMSGCStatsKind,
1.8618 + "Wrong gc statistics type");
1.8619 + counters->update_counters(gc_stats_l);
1.8620 + }
1.8621 +}
1.8622 +
1.8623 +// The desired expansion delta is computed so that:
1.8624 +// . desired free percentage or greater is used
1.8625 +void ASConcurrentMarkSweepGeneration::compute_new_size() {
1.8626 + assert_locked_or_safepoint(Heap_lock);
1.8627 +
1.8628 + GenCollectedHeap* gch = (GenCollectedHeap*) GenCollectedHeap::heap();
1.8629 +
1.8630 + // If incremental collection failed, we just want to expand
1.8631 + // to the limit.
1.8632 + if (incremental_collection_failed()) {
1.8633 + clear_incremental_collection_failed();
1.8634 + grow_to_reserved();
1.8635 + return;
1.8636 + }
1.8637 +
1.8638 + assert(UseAdaptiveSizePolicy, "Should be using adaptive sizing");
1.8639 +
1.8640 + assert(gch->kind() == CollectedHeap::GenCollectedHeap,
1.8641 + "Wrong type of heap");
1.8642 + int prev_level = level() - 1;
1.8643 + assert(prev_level >= 0, "The cms generation is the lowest generation");
1.8644 + Generation* prev_gen = gch->get_gen(prev_level);
1.8645 + assert(prev_gen->kind() == Generation::ASParNew,
1.8646 + "Wrong type of young generation");
1.8647 + ParNewGeneration* younger_gen = (ParNewGeneration*) prev_gen;
1.8648 + size_t cur_eden = younger_gen->eden()->capacity();
1.8649 + CMSAdaptiveSizePolicy* size_policy = cms_size_policy();
1.8650 + size_t cur_promo = free();
1.8651 + size_policy->compute_tenured_generation_free_space(cur_promo,
1.8652 + max_available(),
1.8653 + cur_eden);
1.8654 + resize(cur_promo, size_policy->promo_size());
1.8655 +
1.8656 + // Record the new size of the space in the cms generation
1.8657 + // that is available for promotions. This is temporary.
1.8658 + // It should be the desired promo size.
1.8659 + size_policy->avg_cms_promo()->sample(free());
1.8660 + size_policy->avg_old_live()->sample(used());
1.8661 +
1.8662 + if (UsePerfData) {
1.8663 + CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
1.8664 + counters->update_cms_capacity_counter(capacity());
1.8665 + }
1.8666 +}
1.8667 +
1.8668 +void ASConcurrentMarkSweepGeneration::shrink_by(size_t desired_bytes) {
1.8669 + assert_locked_or_safepoint(Heap_lock);
1.8670 + assert_lock_strong(freelistLock());
1.8671 + HeapWord* old_end = _cmsSpace->end();
1.8672 + HeapWord* unallocated_start = _cmsSpace->unallocated_block();
1.8673 + assert(old_end >= unallocated_start, "Miscalculation of unallocated_start");
1.8674 + FreeChunk* chunk_at_end = find_chunk_at_end();
1.8675 + if (chunk_at_end == NULL) {
1.8676 + // No room to shrink
1.8677 + if (PrintGCDetails && Verbose) {
1.8678 + gclog_or_tty->print_cr("No room to shrink: old_end "
1.8679 + PTR_FORMAT " unallocated_start " PTR_FORMAT
1.8680 + " chunk_at_end " PTR_FORMAT,
1.8681 + old_end, unallocated_start, chunk_at_end);
1.8682 + }
1.8683 + return;
1.8684 + } else {
1.8685 +
1.8686 + // Find the chunk at the end of the space and determine
1.8687 + // how much it can be shrunk.
1.8688 + size_t shrinkable_size_in_bytes = chunk_at_end->size();
1.8689 + size_t aligned_shrinkable_size_in_bytes =
1.8690 + align_size_down(shrinkable_size_in_bytes, os::vm_page_size());
1.8691 + assert(unallocated_start <= chunk_at_end->end(),
1.8692 + "Inconsistent chunk at end of space");
1.8693 + size_t bytes = MIN2(desired_bytes, aligned_shrinkable_size_in_bytes);
1.8694 + size_t word_size_before = heap_word_size(_virtual_space.committed_size());
1.8695 +
1.8696 + // Shrink the underlying space
1.8697 + _virtual_space.shrink_by(bytes);
1.8698 + if (PrintGCDetails && Verbose) {
1.8699 + gclog_or_tty->print_cr("ConcurrentMarkSweepGeneration::shrink_by:"
1.8700 + " desired_bytes " SIZE_FORMAT
1.8701 + " shrinkable_size_in_bytes " SIZE_FORMAT
1.8702 + " aligned_shrinkable_size_in_bytes " SIZE_FORMAT
1.8703 + " bytes " SIZE_FORMAT,
1.8704 + desired_bytes, shrinkable_size_in_bytes,
1.8705 + aligned_shrinkable_size_in_bytes, bytes);
1.8706 + gclog_or_tty->print_cr(" old_end " SIZE_FORMAT
1.8707 + " unallocated_start " SIZE_FORMAT,
1.8708 + old_end, unallocated_start);
1.8709 + }
1.8710 +
1.8711 + // If the space did shrink (shrinking is not guaranteed),
1.8712 + // shrink the chunk at the end by the appropriate amount.
1.8713 + if (((HeapWord*)_virtual_space.high()) < old_end) {
1.8714 + size_t new_word_size =
1.8715 + heap_word_size(_virtual_space.committed_size());
1.8716 +
1.8717 + // Have to remove the chunk from the dictionary because it is changing
1.8718 + // size and might be someplace elsewhere in the dictionary.
1.8719 +
1.8720 + // Get the chunk at end, shrink it, and put it
1.8721 + // back.
1.8722 + _cmsSpace->removeChunkFromDictionary(chunk_at_end);
1.8723 + size_t word_size_change = word_size_before - new_word_size;
1.8724 + size_t chunk_at_end_old_size = chunk_at_end->size();
1.8725 + assert(chunk_at_end_old_size >= word_size_change,
1.8726 + "Shrink is too large");
1.8727 + chunk_at_end->setSize(chunk_at_end_old_size -
1.8728 + word_size_change);
1.8729 + _cmsSpace->freed((HeapWord*) chunk_at_end->end(),
1.8730 + word_size_change);
1.8731 +
1.8732 + _cmsSpace->returnChunkToDictionary(chunk_at_end);
1.8733 +
1.8734 + MemRegion mr(_cmsSpace->bottom(), new_word_size);
1.8735 + _bts->resize(new_word_size); // resize the block offset shared array
1.8736 + Universe::heap()->barrier_set()->resize_covered_region(mr);
1.8737 + _cmsSpace->assert_locked();
1.8738 + _cmsSpace->set_end((HeapWord*)_virtual_space.high());
1.8739 +
1.8740 + NOT_PRODUCT(_cmsSpace->dictionary()->verify());
1.8741 +
1.8742 + // update the space and generation capacity counters
1.8743 + if (UsePerfData) {
1.8744 + _space_counters->update_capacity();
1.8745 + _gen_counters->update_all();
1.8746 + }
1.8747 +
1.8748 + if (Verbose && PrintGCDetails) {
1.8749 + size_t new_mem_size = _virtual_space.committed_size();
1.8750 + size_t old_mem_size = new_mem_size + bytes;
1.8751 + gclog_or_tty->print_cr("Shrinking %s from %ldK by %ldK to %ldK",
1.8752 + name(), old_mem_size/K, bytes/K, new_mem_size/K);
1.8753 + }
1.8754 + }
1.8755 +
1.8756 + assert(_cmsSpace->unallocated_block() <= _cmsSpace->end(),
1.8757 + "Inconsistency at end of space");
1.8758 + assert(chunk_at_end->end() == _cmsSpace->end(),
1.8759 + "Shrinking is inconsistent");
1.8760 + return;
1.8761 + }
1.8762 +}
1.8763 +
1.8764 +// Transfer some number of overflown objects to usual marking
1.8765 +// stack. Return true if some objects were transferred.
1.8766 +bool MarkRefsIntoAndScanClosure::take_from_overflow_list() {
1.8767 + size_t num = MIN2((size_t)_mark_stack->capacity()/4,
1.8768 + (size_t)ParGCDesiredObjsFromOverflowList);
1.8769 +
1.8770 + bool res = _collector->take_from_overflow_list(num, _mark_stack);
1.8771 + assert(_collector->overflow_list_is_empty() || res,
1.8772 + "If list is not empty, we should have taken something");
1.8773 + assert(!res || !_mark_stack->isEmpty(),
1.8774 + "If we took something, it should now be on our stack");
1.8775 + return res;
1.8776 +}
1.8777 +
1.8778 +size_t MarkDeadObjectsClosure::do_blk(HeapWord* addr) {
1.8779 + size_t res = _sp->block_size_no_stall(addr, _collector);
1.8780 + assert(res != 0, "Should always be able to compute a size");
1.8781 + if (_sp->block_is_obj(addr)) {
1.8782 + if (_live_bit_map->isMarked(addr)) {
1.8783 + // It can't have been dead in a previous cycle
1.8784 + guarantee(!_dead_bit_map->isMarked(addr), "No resurrection!");
1.8785 + } else {
1.8786 + _dead_bit_map->mark(addr); // mark the dead object
1.8787 + }
1.8788 + }
1.8789 + return res;
1.8790 +}
