1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Wed Apr 27 01:25:04 2016 +0800
1.3 @@ -0,0 +1,9568 @@
1.4 +/*
1.5 + * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
1.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
1.7 + *
1.8 + * This code is free software; you can redistribute it and/or modify it
1.9 + * under the terms of the GNU General Public License version 2 only, as
1.10 + * published by the Free Software Foundation.
1.11 + *
1.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
1.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1.14 + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
1.15 + * version 2 for more details (a copy is included in the LICENSE file that
1.16 + * accompanied this code).
1.17 + *
1.18 + * You should have received a copy of the GNU General Public License version
1.19 + * 2 along with this work; if not, write to the Free Software Foundation,
1.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
1.21 + *
1.22 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
1.23 + * or visit www.oracle.com if you need additional information or have any
1.24 + * questions.
1.25 + *
1.26 + */
1.27 +
1.28 +#include "precompiled.hpp"
1.29 +#include "classfile/classLoaderData.hpp"
1.30 +#include "classfile/symbolTable.hpp"
1.31 +#include "classfile/systemDictionary.hpp"
1.32 +#include "code/codeCache.hpp"
1.33 +#include "gc_implementation/concurrentMarkSweep/cmsAdaptiveSizePolicy.hpp"
1.34 +#include "gc_implementation/concurrentMarkSweep/cmsCollectorPolicy.hpp"
1.35 +#include "gc_implementation/concurrentMarkSweep/cmsGCAdaptivePolicyCounters.hpp"
1.36 +#include "gc_implementation/concurrentMarkSweep/cmsOopClosures.inline.hpp"
1.37 +#include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp"
1.38 +#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.inline.hpp"
1.39 +#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp"
1.40 +#include "gc_implementation/concurrentMarkSweep/vmCMSOperations.hpp"
1.41 +#include "gc_implementation/parNew/parNewGeneration.hpp"
1.42 +#include "gc_implementation/shared/collectorCounters.hpp"
1.43 +#include "gc_implementation/shared/gcTimer.hpp"
1.44 +#include "gc_implementation/shared/gcTrace.hpp"
1.45 +#include "gc_implementation/shared/gcTraceTime.hpp"
1.46 +#include "gc_implementation/shared/isGCActiveMark.hpp"
1.47 +#include "gc_interface/collectedHeap.inline.hpp"
1.48 +#include "memory/allocation.hpp"
1.49 +#include "memory/cardTableRS.hpp"
1.50 +#include "memory/collectorPolicy.hpp"
1.51 +#include "memory/gcLocker.inline.hpp"
1.52 +#include "memory/genCollectedHeap.hpp"
1.53 +#include "memory/genMarkSweep.hpp"
1.54 +#include "memory/genOopClosures.inline.hpp"
1.55 +#include "memory/iterator.hpp"
1.56 +#include "memory/padded.hpp"
1.57 +#include "memory/referencePolicy.hpp"
1.58 +#include "memory/resourceArea.hpp"
1.59 +#include "memory/tenuredGeneration.hpp"
1.60 +#include "oops/oop.inline.hpp"
1.61 +#include "prims/jvmtiExport.hpp"
1.62 +#include "runtime/globals_extension.hpp"
1.63 +#include "runtime/handles.inline.hpp"
1.64 +#include "runtime/java.hpp"
1.65 +#include "runtime/vmThread.hpp"
1.66 +#include "services/memoryService.hpp"
1.67 +#include "services/runtimeService.hpp"
1.68 +
1.69 +PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
1.70 +
1.71 +// statics
1.72 +CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;
1.73 +bool CMSCollector::_full_gc_requested = false;
1.74 +GCCause::Cause CMSCollector::_full_gc_cause = GCCause::_no_gc;
1.75 +
1.76 +//////////////////////////////////////////////////////////////////
1.77 +// In support of CMS/VM thread synchronization
1.78 +//////////////////////////////////////////////////////////////////
1.79 +// We split use of the CGC_lock into 2 "levels".
1.80 +// The low-level locking is of the usual CGC_lock monitor. We introduce
1.81 +// a higher level "token" (hereafter "CMS token") built on top of the
1.82 +// low level monitor (hereafter "CGC lock").
1.83 +// The token-passing protocol gives priority to the VM thread. The
1.84 +// CMS-lock doesn't provide any fairness guarantees, but clients
1.85 +// should ensure that it is only held for very short, bounded
1.86 +// durations.
1.87 +//
1.88 +// When either of the CMS thread or the VM thread is involved in
1.89 +// collection operations during which it does not want the other
1.90 +// thread to interfere, it obtains the CMS token.
1.91 +//
1.92 +// If either thread tries to get the token while the other has
1.93 +// it, that thread waits. However, if the VM thread and CMS thread
1.94 +// both want the token, then the VM thread gets priority while the
1.95 +// CMS thread waits. This ensures, for instance, that the "concurrent"
1.96 +// phases of the CMS thread's work do not block out the VM thread
1.97 +// for long periods of time as the CMS thread continues to hog
1.98 +// the token. (See bug 4616232).
1.99 +//
1.100 +// The baton-passing functions are, however, controlled by the
1.101 +// flags _foregroundGCShouldWait and _foregroundGCIsActive,
1.102 +// and here the low-level CMS lock, not the high level token,
1.103 +// ensures mutual exclusion.
1.104 +//
1.105 +// Two important conditions that we have to satisfy:
1.106 +// 1. if a thread does a low-level wait on the CMS lock, then it
1.107 +// relinquishes the CMS token if it were holding that token
1.108 +// when it acquired the low-level CMS lock.
1.109 +// 2. any low-level notifications on the low-level lock
1.110 +// should only be sent when a thread has relinquished the token.
1.111 +//
1.112 +// In the absence of either property, we'd have potential deadlock.
1.113 +//
1.114 +// We protect each of the CMS (concurrent and sequential) phases
1.115 +// with the CMS _token_, not the CMS _lock_.
1.116 +//
1.117 +// The only code protected by CMS lock is the token acquisition code
1.118 +// itself, see ConcurrentMarkSweepThread::[de]synchronize(), and the
1.119 +// baton-passing code.
1.120 +//
1.121 +// Unfortunately, i couldn't come up with a good abstraction to factor and
1.122 +// hide the naked CGC_lock manipulation in the baton-passing code
1.123 +// further below. That's something we should try to do. Also, the proof
1.124 +// of correctness of this 2-level locking scheme is far from obvious,
1.125 +// and potentially quite slippery. We have an uneasy supsicion, for instance,
1.126 +// that there may be a theoretical possibility of delay/starvation in the
1.127 +// low-level lock/wait/notify scheme used for the baton-passing because of
1.128 +// potential intereference with the priority scheme embodied in the
1.129 +// CMS-token-passing protocol. See related comments at a CGC_lock->wait()
1.130 +// invocation further below and marked with "XXX 20011219YSR".
1.131 +// Indeed, as we note elsewhere, this may become yet more slippery
1.132 +// in the presence of multiple CMS and/or multiple VM threads. XXX
1.133 +
1.134 +class CMSTokenSync: public StackObj {
1.135 + private:
1.136 + bool _is_cms_thread;
1.137 + public:
1.138 + CMSTokenSync(bool is_cms_thread):
1.139 + _is_cms_thread(is_cms_thread) {
1.140 + assert(is_cms_thread == Thread::current()->is_ConcurrentGC_thread(),
1.141 + "Incorrect argument to constructor");
1.142 + ConcurrentMarkSweepThread::synchronize(_is_cms_thread);
1.143 + }
1.144 +
1.145 + ~CMSTokenSync() {
1.146 + assert(_is_cms_thread ?
1.147 + ConcurrentMarkSweepThread::cms_thread_has_cms_token() :
1.148 + ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1.149 + "Incorrect state");
1.150 + ConcurrentMarkSweepThread::desynchronize(_is_cms_thread);
1.151 + }
1.152 +};
1.153 +
1.154 +// Convenience class that does a CMSTokenSync, and then acquires
1.155 +// upto three locks.
1.156 +class CMSTokenSyncWithLocks: public CMSTokenSync {
1.157 + private:
1.158 + // Note: locks are acquired in textual declaration order
1.159 + // and released in the opposite order
1.160 + MutexLockerEx _locker1, _locker2, _locker3;
1.161 + public:
1.162 + CMSTokenSyncWithLocks(bool is_cms_thread, Mutex* mutex1,
1.163 + Mutex* mutex2 = NULL, Mutex* mutex3 = NULL):
1.164 + CMSTokenSync(is_cms_thread),
1.165 + _locker1(mutex1, Mutex::_no_safepoint_check_flag),
1.166 + _locker2(mutex2, Mutex::_no_safepoint_check_flag),
1.167 + _locker3(mutex3, Mutex::_no_safepoint_check_flag)
1.168 + { }
1.169 +};
1.170 +
1.171 +
1.172 +// Wrapper class to temporarily disable icms during a foreground cms collection.
1.173 +class ICMSDisabler: public StackObj {
1.174 + public:
1.175 + // The ctor disables icms and wakes up the thread so it notices the change;
1.176 + // the dtor re-enables icms. Note that the CMSCollector methods will check
1.177 + // CMSIncrementalMode.
1.178 + ICMSDisabler() { CMSCollector::disable_icms(); CMSCollector::start_icms(); }
1.179 + ~ICMSDisabler() { CMSCollector::enable_icms(); }
1.180 +};
1.181 +
1.182 +//////////////////////////////////////////////////////////////////
1.183 +// Concurrent Mark-Sweep Generation /////////////////////////////
1.184 +//////////////////////////////////////////////////////////////////
1.185 +
1.186 +NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;)
1.187 +
1.188 +// This struct contains per-thread things necessary to support parallel
1.189 +// young-gen collection.
1.190 +class CMSParGCThreadState: public CHeapObj<mtGC> {
1.191 + public:
1.192 + CFLS_LAB lab;
1.193 + PromotionInfo promo;
1.194 +
1.195 + // Constructor.
1.196 + CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) {
1.197 + promo.setSpace(cfls);
1.198 + }
1.199 +};
1.200 +
1.201 +ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration(
1.202 + ReservedSpace rs, size_t initial_byte_size, int level,
1.203 + CardTableRS* ct, bool use_adaptive_freelists,
1.204 + FreeBlockDictionary<FreeChunk>::DictionaryChoice dictionaryChoice) :
1.205 + CardGeneration(rs, initial_byte_size, level, ct),
1.206 + _dilatation_factor(((double)MinChunkSize)/((double)(CollectedHeap::min_fill_size()))),
1.207 + _debug_collection_type(Concurrent_collection_type),
1.208 + _did_compact(false)
1.209 +{
1.210 + HeapWord* bottom = (HeapWord*) _virtual_space.low();
1.211 + HeapWord* end = (HeapWord*) _virtual_space.high();
1.212 +
1.213 + _direct_allocated_words = 0;
1.214 + NOT_PRODUCT(
1.215 + _numObjectsPromoted = 0;
1.216 + _numWordsPromoted = 0;
1.217 + _numObjectsAllocated = 0;
1.218 + _numWordsAllocated = 0;
1.219 + )
1.220 +
1.221 + _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end),
1.222 + use_adaptive_freelists,
1.223 + dictionaryChoice);
1.224 + NOT_PRODUCT(debug_cms_space = _cmsSpace;)
1.225 + if (_cmsSpace == NULL) {
1.226 + vm_exit_during_initialization(
1.227 + "CompactibleFreeListSpace allocation failure");
1.228 + }
1.229 + _cmsSpace->_gen = this;
1.230 +
1.231 + _gc_stats = new CMSGCStats();
1.232 +
1.233 + // Verify the assumption that FreeChunk::_prev and OopDesc::_klass
1.234 + // offsets match. The ability to tell free chunks from objects
1.235 + // depends on this property.
1.236 + debug_only(
1.237 + FreeChunk* junk = NULL;
1.238 + assert(UseCompressedClassPointers ||
1.239 + junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
1.240 + "Offset of FreeChunk::_prev within FreeChunk must match"
1.241 + " that of OopDesc::_klass within OopDesc");
1.242 + )
1.243 + if (CollectedHeap::use_parallel_gc_threads()) {
1.244 + typedef CMSParGCThreadState* CMSParGCThreadStatePtr;
1.245 + _par_gc_thread_states =
1.246 + NEW_C_HEAP_ARRAY(CMSParGCThreadStatePtr, ParallelGCThreads, mtGC);
1.247 + if (_par_gc_thread_states == NULL) {
1.248 + vm_exit_during_initialization("Could not allocate par gc structs");
1.249 + }
1.250 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.251 + _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace());
1.252 + if (_par_gc_thread_states[i] == NULL) {
1.253 + vm_exit_during_initialization("Could not allocate par gc structs");
1.254 + }
1.255 + }
1.256 + } else {
1.257 + _par_gc_thread_states = NULL;
1.258 + }
1.259 + _incremental_collection_failed = false;
1.260 + // The "dilatation_factor" is the expansion that can occur on
1.261 + // account of the fact that the minimum object size in the CMS
1.262 + // generation may be larger than that in, say, a contiguous young
1.263 + // generation.
1.264 + // Ideally, in the calculation below, we'd compute the dilatation
1.265 + // factor as: MinChunkSize/(promoting_gen's min object size)
1.266 + // Since we do not have such a general query interface for the
1.267 + // promoting generation, we'll instead just use the mimimum
1.268 + // object size (which today is a header's worth of space);
1.269 + // note that all arithmetic is in units of HeapWords.
1.270 + assert(MinChunkSize >= CollectedHeap::min_fill_size(), "just checking");
1.271 + assert(_dilatation_factor >= 1.0, "from previous assert");
1.272 +}
1.273 +
1.274 +
1.275 +// The field "_initiating_occupancy" represents the occupancy percentage
1.276 +// at which we trigger a new collection cycle. Unless explicitly specified
1.277 +// via CMSInitiatingOccupancyFraction (argument "io" below), it
1.278 +// is calculated by:
1.279 +//
1.280 +// Let "f" be MinHeapFreeRatio in
1.281 +//
1.282 +// _intiating_occupancy = 100-f +
1.283 +// f * (CMSTriggerRatio/100)
1.284 +// where CMSTriggerRatio is the argument "tr" below.
1.285 +//
1.286 +// That is, if we assume the heap is at its desired maximum occupancy at the
1.287 +// end of a collection, we let CMSTriggerRatio of the (purported) free
1.288 +// space be allocated before initiating a new collection cycle.
1.289 +//
1.290 +void ConcurrentMarkSweepGeneration::init_initiating_occupancy(intx io, uintx tr) {
1.291 + assert(io <= 100 && tr <= 100, "Check the arguments");
1.292 + if (io >= 0) {
1.293 + _initiating_occupancy = (double)io / 100.0;
1.294 + } else {
1.295 + _initiating_occupancy = ((100 - MinHeapFreeRatio) +
1.296 + (double)(tr * MinHeapFreeRatio) / 100.0)
1.297 + / 100.0;
1.298 + }
1.299 +}
1.300 +
1.301 +void ConcurrentMarkSweepGeneration::ref_processor_init() {
1.302 + assert(collector() != NULL, "no collector");
1.303 + collector()->ref_processor_init();
1.304 +}
1.305 +
1.306 +void CMSCollector::ref_processor_init() {
1.307 + if (_ref_processor == NULL) {
1.308 + // Allocate and initialize a reference processor
1.309 + _ref_processor =
1.310 + new ReferenceProcessor(_span, // span
1.311 + (ParallelGCThreads > 1) && ParallelRefProcEnabled, // mt processing
1.312 + (int) ParallelGCThreads, // mt processing degree
1.313 + _cmsGen->refs_discovery_is_mt(), // mt discovery
1.314 + (int) MAX2(ConcGCThreads, ParallelGCThreads), // mt discovery degree
1.315 + _cmsGen->refs_discovery_is_atomic(), // discovery is not atomic
1.316 + &_is_alive_closure); // closure for liveness info
1.317 + // Initialize the _ref_processor field of CMSGen
1.318 + _cmsGen->set_ref_processor(_ref_processor);
1.319 +
1.320 + }
1.321 +}
1.322 +
1.323 +CMSAdaptiveSizePolicy* CMSCollector::size_policy() {
1.324 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.325 + assert(gch->kind() == CollectedHeap::GenCollectedHeap,
1.326 + "Wrong type of heap");
1.327 + CMSAdaptiveSizePolicy* sp = (CMSAdaptiveSizePolicy*)
1.328 + gch->gen_policy()->size_policy();
1.329 + assert(sp->is_gc_cms_adaptive_size_policy(),
1.330 + "Wrong type of size policy");
1.331 + return sp;
1.332 +}
1.333 +
1.334 +CMSGCAdaptivePolicyCounters* CMSCollector::gc_adaptive_policy_counters() {
1.335 + CMSGCAdaptivePolicyCounters* results =
1.336 + (CMSGCAdaptivePolicyCounters*) collector_policy()->counters();
1.337 + assert(
1.338 + results->kind() == GCPolicyCounters::CMSGCAdaptivePolicyCountersKind,
1.339 + "Wrong gc policy counter kind");
1.340 + return results;
1.341 +}
1.342 +
1.343 +
1.344 +void ConcurrentMarkSweepGeneration::initialize_performance_counters() {
1.345 +
1.346 + const char* gen_name = "old";
1.347 +
1.348 + // Generation Counters - generation 1, 1 subspace
1.349 + _gen_counters = new GenerationCounters(gen_name, 1, 1, &_virtual_space);
1.350 +
1.351 + _space_counters = new GSpaceCounters(gen_name, 0,
1.352 + _virtual_space.reserved_size(),
1.353 + this, _gen_counters);
1.354 +}
1.355 +
1.356 +CMSStats::CMSStats(ConcurrentMarkSweepGeneration* cms_gen, unsigned int alpha):
1.357 + _cms_gen(cms_gen)
1.358 +{
1.359 + assert(alpha <= 100, "bad value");
1.360 + _saved_alpha = alpha;
1.361 +
1.362 + // Initialize the alphas to the bootstrap value of 100.
1.363 + _gc0_alpha = _cms_alpha = 100;
1.364 +
1.365 + _cms_begin_time.update();
1.366 + _cms_end_time.update();
1.367 +
1.368 + _gc0_duration = 0.0;
1.369 + _gc0_period = 0.0;
1.370 + _gc0_promoted = 0;
1.371 +
1.372 + _cms_duration = 0.0;
1.373 + _cms_period = 0.0;
1.374 + _cms_allocated = 0;
1.375 +
1.376 + _cms_used_at_gc0_begin = 0;
1.377 + _cms_used_at_gc0_end = 0;
1.378 + _allow_duty_cycle_reduction = false;
1.379 + _valid_bits = 0;
1.380 + _icms_duty_cycle = CMSIncrementalDutyCycle;
1.381 +}
1.382 +
1.383 +double CMSStats::cms_free_adjustment_factor(size_t free) const {
1.384 + // TBD: CR 6909490
1.385 + return 1.0;
1.386 +}
1.387 +
1.388 +void CMSStats::adjust_cms_free_adjustment_factor(bool fail, size_t free) {
1.389 +}
1.390 +
1.391 +// If promotion failure handling is on use
1.392 +// the padded average size of the promotion for each
1.393 +// young generation collection.
1.394 +double CMSStats::time_until_cms_gen_full() const {
1.395 + size_t cms_free = _cms_gen->cmsSpace()->free();
1.396 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.397 + size_t expected_promotion = MIN2(gch->get_gen(0)->capacity(),
1.398 + (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average());
1.399 + if (cms_free > expected_promotion) {
1.400 + // Start a cms collection if there isn't enough space to promote
1.401 + // for the next minor collection. Use the padded average as
1.402 + // a safety factor.
1.403 + cms_free -= expected_promotion;
1.404 +
1.405 + // Adjust by the safety factor.
1.406 + double cms_free_dbl = (double)cms_free;
1.407 + double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor)/100.0;
1.408 + // Apply a further correction factor which tries to adjust
1.409 + // for recent occurance of concurrent mode failures.
1.410 + cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free);
1.411 + cms_free_dbl = cms_free_dbl * cms_adjustment;
1.412 +
1.413 + if (PrintGCDetails && Verbose) {
1.414 + gclog_or_tty->print_cr("CMSStats::time_until_cms_gen_full: cms_free "
1.415 + SIZE_FORMAT " expected_promotion " SIZE_FORMAT,
1.416 + cms_free, expected_promotion);
1.417 + gclog_or_tty->print_cr(" cms_free_dbl %f cms_consumption_rate %f",
1.418 + cms_free_dbl, cms_consumption_rate() + 1.0);
1.419 + }
1.420 + // Add 1 in case the consumption rate goes to zero.
1.421 + return cms_free_dbl / (cms_consumption_rate() + 1.0);
1.422 + }
1.423 + return 0.0;
1.424 +}
1.425 +
1.426 +// Compare the duration of the cms collection to the
1.427 +// time remaining before the cms generation is empty.
1.428 +// Note that the time from the start of the cms collection
1.429 +// to the start of the cms sweep (less than the total
1.430 +// duration of the cms collection) can be used. This
1.431 +// has been tried and some applications experienced
1.432 +// promotion failures early in execution. This was
1.433 +// possibly because the averages were not accurate
1.434 +// enough at the beginning.
1.435 +double CMSStats::time_until_cms_start() const {
1.436 + // We add "gc0_period" to the "work" calculation
1.437 + // below because this query is done (mostly) at the
1.438 + // end of a scavenge, so we need to conservatively
1.439 + // account for that much possible delay
1.440 + // in the query so as to avoid concurrent mode failures
1.441 + // due to starting the collection just a wee bit too
1.442 + // late.
1.443 + double work = cms_duration() + gc0_period();
1.444 + double deadline = time_until_cms_gen_full();
1.445 + // If a concurrent mode failure occurred recently, we want to be
1.446 + // more conservative and halve our expected time_until_cms_gen_full()
1.447 + if (work > deadline) {
1.448 + if (Verbose && PrintGCDetails) {
1.449 + gclog_or_tty->print(
1.450 + " CMSCollector: collect because of anticipated promotion "
1.451 + "before full %3.7f + %3.7f > %3.7f ", cms_duration(),
1.452 + gc0_period(), time_until_cms_gen_full());
1.453 + }
1.454 + return 0.0;
1.455 + }
1.456 + return work - deadline;
1.457 +}
1.458 +
1.459 +// Return a duty cycle based on old_duty_cycle and new_duty_cycle, limiting the
1.460 +// amount of change to prevent wild oscillation.
1.461 +unsigned int CMSStats::icms_damped_duty_cycle(unsigned int old_duty_cycle,
1.462 + unsigned int new_duty_cycle) {
1.463 + assert(old_duty_cycle <= 100, "bad input value");
1.464 + assert(new_duty_cycle <= 100, "bad input value");
1.465 +
1.466 + // Note: use subtraction with caution since it may underflow (values are
1.467 + // unsigned). Addition is safe since we're in the range 0-100.
1.468 + unsigned int damped_duty_cycle = new_duty_cycle;
1.469 + if (new_duty_cycle < old_duty_cycle) {
1.470 + const unsigned int largest_delta = MAX2(old_duty_cycle / 4, 5U);
1.471 + if (new_duty_cycle + largest_delta < old_duty_cycle) {
1.472 + damped_duty_cycle = old_duty_cycle - largest_delta;
1.473 + }
1.474 + } else if (new_duty_cycle > old_duty_cycle) {
1.475 + const unsigned int largest_delta = MAX2(old_duty_cycle / 4, 15U);
1.476 + if (new_duty_cycle > old_duty_cycle + largest_delta) {
1.477 + damped_duty_cycle = MIN2(old_duty_cycle + largest_delta, 100U);
1.478 + }
1.479 + }
1.480 + assert(damped_duty_cycle <= 100, "invalid duty cycle computed");
1.481 +
1.482 + if (CMSTraceIncrementalPacing) {
1.483 + gclog_or_tty->print(" [icms_damped_duty_cycle(%d,%d) = %d] ",
1.484 + old_duty_cycle, new_duty_cycle, damped_duty_cycle);
1.485 + }
1.486 + return damped_duty_cycle;
1.487 +}
1.488 +
1.489 +unsigned int CMSStats::icms_update_duty_cycle_impl() {
1.490 + assert(CMSIncrementalPacing && valid(),
1.491 + "should be handled in icms_update_duty_cycle()");
1.492 +
1.493 + double cms_time_so_far = cms_timer().seconds();
1.494 + double scaled_duration = cms_duration_per_mb() * _cms_used_at_gc0_end / M;
1.495 + double scaled_duration_remaining = fabsd(scaled_duration - cms_time_so_far);
1.496 +
1.497 + // Avoid division by 0.
1.498 + double time_until_full = MAX2(time_until_cms_gen_full(), 0.01);
1.499 + double duty_cycle_dbl = 100.0 * scaled_duration_remaining / time_until_full;
1.500 +
1.501 + unsigned int new_duty_cycle = MIN2((unsigned int)duty_cycle_dbl, 100U);
1.502 + if (new_duty_cycle > _icms_duty_cycle) {
1.503 + // Avoid very small duty cycles (1 or 2); 0 is allowed.
1.504 + if (new_duty_cycle > 2) {
1.505 + _icms_duty_cycle = icms_damped_duty_cycle(_icms_duty_cycle,
1.506 + new_duty_cycle);
1.507 + }
1.508 + } else if (_allow_duty_cycle_reduction) {
1.509 + // The duty cycle is reduced only once per cms cycle (see record_cms_end()).
1.510 + new_duty_cycle = icms_damped_duty_cycle(_icms_duty_cycle, new_duty_cycle);
1.511 + // Respect the minimum duty cycle.
1.512 + unsigned int min_duty_cycle = (unsigned int)CMSIncrementalDutyCycleMin;
1.513 + _icms_duty_cycle = MAX2(new_duty_cycle, min_duty_cycle);
1.514 + }
1.515 +
1.516 + if (PrintGCDetails || CMSTraceIncrementalPacing) {
1.517 + gclog_or_tty->print(" icms_dc=%d ", _icms_duty_cycle);
1.518 + }
1.519 +
1.520 + _allow_duty_cycle_reduction = false;
1.521 + return _icms_duty_cycle;
1.522 +}
1.523 +
1.524 +#ifndef PRODUCT
1.525 +void CMSStats::print_on(outputStream *st) const {
1.526 + st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);
1.527 + st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,
1.528 + gc0_duration(), gc0_period(), gc0_promoted());
1.529 + st->print(",cms_dur=%g,cms_dur_per_mb=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,
1.530 + cms_duration(), cms_duration_per_mb(),
1.531 + cms_period(), cms_allocated());
1.532 + st->print(",cms_since_beg=%g,cms_since_end=%g",
1.533 + cms_time_since_begin(), cms_time_since_end());
1.534 + st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,
1.535 + _cms_used_at_gc0_begin, _cms_used_at_gc0_end);
1.536 + if (CMSIncrementalMode) {
1.537 + st->print(",dc=%d", icms_duty_cycle());
1.538 + }
1.539 +
1.540 + if (valid()) {
1.541 + st->print(",promo_rate=%g,cms_alloc_rate=%g",
1.542 + promotion_rate(), cms_allocation_rate());
1.543 + st->print(",cms_consumption_rate=%g,time_until_full=%g",
1.544 + cms_consumption_rate(), time_until_cms_gen_full());
1.545 + }
1.546 + st->print(" ");
1.547 +}
1.548 +#endif // #ifndef PRODUCT
1.549 +
1.550 +CMSCollector::CollectorState CMSCollector::_collectorState =
1.551 + CMSCollector::Idling;
1.552 +bool CMSCollector::_foregroundGCIsActive = false;
1.553 +bool CMSCollector::_foregroundGCShouldWait = false;
1.554 +
1.555 +CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
1.556 + CardTableRS* ct,
1.557 + ConcurrentMarkSweepPolicy* cp):
1.558 + _cmsGen(cmsGen),
1.559 + _ct(ct),
1.560 + _ref_processor(NULL), // will be set later
1.561 + _conc_workers(NULL), // may be set later
1.562 + _abort_preclean(false),
1.563 + _start_sampling(false),
1.564 + _between_prologue_and_epilogue(false),
1.565 + _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
1.566 + _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize),
1.567 + -1 /* lock-free */, "No_lock" /* dummy */),
1.568 + _modUnionClosure(&_modUnionTable),
1.569 + _modUnionClosurePar(&_modUnionTable),
1.570 + // Adjust my span to cover old (cms) gen
1.571 + _span(cmsGen->reserved()),
1.572 + // Construct the is_alive_closure with _span & markBitMap
1.573 + _is_alive_closure(_span, &_markBitMap),
1.574 + _restart_addr(NULL),
1.575 + _overflow_list(NULL),
1.576 + _stats(cmsGen),
1.577 + _eden_chunk_lock(new Mutex(Mutex::leaf + 1, "CMS_eden_chunk_lock", true)),
1.578 + _eden_chunk_array(NULL), // may be set in ctor body
1.579 + _eden_chunk_capacity(0), // -- ditto --
1.580 + _eden_chunk_index(0), // -- ditto --
1.581 + _survivor_plab_array(NULL), // -- ditto --
1.582 + _survivor_chunk_array(NULL), // -- ditto --
1.583 + _survivor_chunk_capacity(0), // -- ditto --
1.584 + _survivor_chunk_index(0), // -- ditto --
1.585 + _ser_pmc_preclean_ovflw(0),
1.586 + _ser_kac_preclean_ovflw(0),
1.587 + _ser_pmc_remark_ovflw(0),
1.588 + _par_pmc_remark_ovflw(0),
1.589 + _ser_kac_ovflw(0),
1.590 + _par_kac_ovflw(0),
1.591 +#ifndef PRODUCT
1.592 + _num_par_pushes(0),
1.593 +#endif
1.594 + _collection_count_start(0),
1.595 + _verifying(false),
1.596 + _icms_start_limit(NULL),
1.597 + _icms_stop_limit(NULL),
1.598 + _verification_mark_bm(0, Mutex::leaf + 1, "CMS_verification_mark_bm_lock"),
1.599 + _completed_initialization(false),
1.600 + _collector_policy(cp),
1.601 + _should_unload_classes(CMSClassUnloadingEnabled),
1.602 + _concurrent_cycles_since_last_unload(0),
1.603 + _roots_scanning_options(SharedHeap::SO_None),
1.604 + _inter_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding),
1.605 + _intra_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding),
1.606 + _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) CMSTracer()),
1.607 + _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
1.608 + _cms_start_registered(false)
1.609 +{
1.610 + if (ExplicitGCInvokesConcurrentAndUnloadsClasses) {
1.611 + ExplicitGCInvokesConcurrent = true;
1.612 + }
1.613 + // Now expand the span and allocate the collection support structures
1.614 + // (MUT, marking bit map etc.) to cover both generations subject to
1.615 + // collection.
1.616 +
1.617 + // For use by dirty card to oop closures.
1.618 + _cmsGen->cmsSpace()->set_collector(this);
1.619 +
1.620 + // Allocate MUT and marking bit map
1.621 + {
1.622 + MutexLockerEx x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag);
1.623 + if (!_markBitMap.allocate(_span)) {
1.624 + warning("Failed to allocate CMS Bit Map");
1.625 + return;
1.626 + }
1.627 + assert(_markBitMap.covers(_span), "_markBitMap inconsistency?");
1.628 + }
1.629 + {
1.630 + _modUnionTable.allocate(_span);
1.631 + assert(_modUnionTable.covers(_span), "_modUnionTable inconsistency?");
1.632 + }
1.633 +
1.634 + if (!_markStack.allocate(MarkStackSize)) {
1.635 + warning("Failed to allocate CMS Marking Stack");
1.636 + return;
1.637 + }
1.638 +
1.639 + // Support for multi-threaded concurrent phases
1.640 + if (CMSConcurrentMTEnabled) {
1.641 + if (FLAG_IS_DEFAULT(ConcGCThreads)) {
1.642 + // just for now
1.643 + FLAG_SET_DEFAULT(ConcGCThreads, (ParallelGCThreads + 3)/4);
1.644 + }
1.645 + if (ConcGCThreads > 1) {
1.646 + _conc_workers = new YieldingFlexibleWorkGang("Parallel CMS Threads",
1.647 + ConcGCThreads, true);
1.648 + if (_conc_workers == NULL) {
1.649 + warning("GC/CMS: _conc_workers allocation failure: "
1.650 + "forcing -CMSConcurrentMTEnabled");
1.651 + CMSConcurrentMTEnabled = false;
1.652 + } else {
1.653 + _conc_workers->initialize_workers();
1.654 + }
1.655 + } else {
1.656 + CMSConcurrentMTEnabled = false;
1.657 + }
1.658 + }
1.659 + if (!CMSConcurrentMTEnabled) {
1.660 + ConcGCThreads = 0;
1.661 + } else {
1.662 + // Turn off CMSCleanOnEnter optimization temporarily for
1.663 + // the MT case where it's not fixed yet; see 6178663.
1.664 + CMSCleanOnEnter = false;
1.665 + }
1.666 + assert((_conc_workers != NULL) == (ConcGCThreads > 1),
1.667 + "Inconsistency");
1.668 +
1.669 + // Parallel task queues; these are shared for the
1.670 + // concurrent and stop-world phases of CMS, but
1.671 + // are not shared with parallel scavenge (ParNew).
1.672 + {
1.673 + uint i;
1.674 + uint num_queues = (uint) MAX2(ParallelGCThreads, ConcGCThreads);
1.675 +
1.676 + if ((CMSParallelRemarkEnabled || CMSConcurrentMTEnabled
1.677 + || ParallelRefProcEnabled)
1.678 + && num_queues > 0) {
1.679 + _task_queues = new OopTaskQueueSet(num_queues);
1.680 + if (_task_queues == NULL) {
1.681 + warning("task_queues allocation failure.");
1.682 + return;
1.683 + }
1.684 + _hash_seed = NEW_C_HEAP_ARRAY(int, num_queues, mtGC);
1.685 + if (_hash_seed == NULL) {
1.686 + warning("_hash_seed array allocation failure");
1.687 + return;
1.688 + }
1.689 +
1.690 + typedef Padded<OopTaskQueue> PaddedOopTaskQueue;
1.691 + for (i = 0; i < num_queues; i++) {
1.692 + PaddedOopTaskQueue *q = new PaddedOopTaskQueue();
1.693 + if (q == NULL) {
1.694 + warning("work_queue allocation failure.");
1.695 + return;
1.696 + }
1.697 + _task_queues->register_queue(i, q);
1.698 + }
1.699 + for (i = 0; i < num_queues; i++) {
1.700 + _task_queues->queue(i)->initialize();
1.701 + _hash_seed[i] = 17; // copied from ParNew
1.702 + }
1.703 + }
1.704 + }
1.705 +
1.706 + _cmsGen ->init_initiating_occupancy(CMSInitiatingOccupancyFraction, CMSTriggerRatio);
1.707 +
1.708 + // Clip CMSBootstrapOccupancy between 0 and 100.
1.709 + _bootstrap_occupancy = ((double)CMSBootstrapOccupancy)/(double)100;
1.710 +
1.711 + _full_gcs_since_conc_gc = 0;
1.712 +
1.713 + // Now tell CMS generations the identity of their collector
1.714 + ConcurrentMarkSweepGeneration::set_collector(this);
1.715 +
1.716 + // Create & start a CMS thread for this CMS collector
1.717 + _cmsThread = ConcurrentMarkSweepThread::start(this);
1.718 + assert(cmsThread() != NULL, "CMS Thread should have been created");
1.719 + assert(cmsThread()->collector() == this,
1.720 + "CMS Thread should refer to this gen");
1.721 + assert(CGC_lock != NULL, "Where's the CGC_lock?");
1.722 +
1.723 + // Support for parallelizing young gen rescan
1.724 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.725 + _young_gen = gch->prev_gen(_cmsGen);
1.726 + if (gch->supports_inline_contig_alloc()) {
1.727 + _top_addr = gch->top_addr();
1.728 + _end_addr = gch->end_addr();
1.729 + assert(_young_gen != NULL, "no _young_gen");
1.730 + _eden_chunk_index = 0;
1.731 + _eden_chunk_capacity = (_young_gen->max_capacity()+CMSSamplingGrain)/CMSSamplingGrain;
1.732 + _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity, mtGC);
1.733 + if (_eden_chunk_array == NULL) {
1.734 + _eden_chunk_capacity = 0;
1.735 + warning("GC/CMS: _eden_chunk_array allocation failure");
1.736 + }
1.737 + }
1.738 + assert(_eden_chunk_array != NULL || _eden_chunk_capacity == 0, "Error");
1.739 +
1.740 + // Support for parallelizing survivor space rescan
1.741 + if ((CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) || CMSParallelInitialMarkEnabled) {
1.742 + const size_t max_plab_samples =
1.743 + ((DefNewGeneration*)_young_gen)->max_survivor_size()/MinTLABSize;
1.744 +
1.745 + _survivor_plab_array = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads, mtGC);
1.746 + _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, 2*max_plab_samples, mtGC);
1.747 + _cursor = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads, mtGC);
1.748 + if (_survivor_plab_array == NULL || _survivor_chunk_array == NULL
1.749 + || _cursor == NULL) {
1.750 + warning("Failed to allocate survivor plab/chunk array");
1.751 + if (_survivor_plab_array != NULL) {
1.752 + FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array, mtGC);
1.753 + _survivor_plab_array = NULL;
1.754 + }
1.755 + if (_survivor_chunk_array != NULL) {
1.756 + FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array, mtGC);
1.757 + _survivor_chunk_array = NULL;
1.758 + }
1.759 + if (_cursor != NULL) {
1.760 + FREE_C_HEAP_ARRAY(size_t, _cursor, mtGC);
1.761 + _cursor = NULL;
1.762 + }
1.763 + } else {
1.764 + _survivor_chunk_capacity = 2*max_plab_samples;
1.765 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.766 + HeapWord** vec = NEW_C_HEAP_ARRAY(HeapWord*, max_plab_samples, mtGC);
1.767 + if (vec == NULL) {
1.768 + warning("Failed to allocate survivor plab array");
1.769 + for (int j = i; j > 0; j--) {
1.770 + FREE_C_HEAP_ARRAY(HeapWord*, _survivor_plab_array[j-1].array(), mtGC);
1.771 + }
1.772 + FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array, mtGC);
1.773 + FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array, mtGC);
1.774 + _survivor_plab_array = NULL;
1.775 + _survivor_chunk_array = NULL;
1.776 + _survivor_chunk_capacity = 0;
1.777 + break;
1.778 + } else {
1.779 + ChunkArray* cur =
1.780 + ::new (&_survivor_plab_array[i]) ChunkArray(vec,
1.781 + max_plab_samples);
1.782 + assert(cur->end() == 0, "Should be 0");
1.783 + assert(cur->array() == vec, "Should be vec");
1.784 + assert(cur->capacity() == max_plab_samples, "Error");
1.785 + }
1.786 + }
1.787 + }
1.788 + }
1.789 + assert( ( _survivor_plab_array != NULL
1.790 + && _survivor_chunk_array != NULL)
1.791 + || ( _survivor_chunk_capacity == 0
1.792 + && _survivor_chunk_index == 0),
1.793 + "Error");
1.794 +
1.795 + NOT_PRODUCT(_overflow_counter = CMSMarkStackOverflowInterval;)
1.796 + _gc_counters = new CollectorCounters("CMS", 1);
1.797 + _completed_initialization = true;
1.798 + _inter_sweep_timer.start(); // start of time
1.799 +}
1.800 +
1.801 +const char* ConcurrentMarkSweepGeneration::name() const {
1.802 + return "concurrent mark-sweep generation";
1.803 +}
1.804 +void ConcurrentMarkSweepGeneration::update_counters() {
1.805 + if (UsePerfData) {
1.806 + _space_counters->update_all();
1.807 + _gen_counters->update_all();
1.808 + }
1.809 +}
1.810 +
1.811 +// this is an optimized version of update_counters(). it takes the
1.812 +// used value as a parameter rather than computing it.
1.813 +//
1.814 +void ConcurrentMarkSweepGeneration::update_counters(size_t used) {
1.815 + if (UsePerfData) {
1.816 + _space_counters->update_used(used);
1.817 + _space_counters->update_capacity();
1.818 + _gen_counters->update_all();
1.819 + }
1.820 +}
1.821 +
1.822 +void ConcurrentMarkSweepGeneration::print() const {
1.823 + Generation::print();
1.824 + cmsSpace()->print();
1.825 +}
1.826 +
1.827 +#ifndef PRODUCT
1.828 +void ConcurrentMarkSweepGeneration::print_statistics() {
1.829 + cmsSpace()->printFLCensus(0);
1.830 +}
1.831 +#endif
1.832 +
1.833 +void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) {
1.834 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.835 + if (PrintGCDetails) {
1.836 + if (Verbose) {
1.837 + gclog_or_tty->print("[%d %s-%s: "SIZE_FORMAT"("SIZE_FORMAT")]",
1.838 + level(), short_name(), s, used(), capacity());
1.839 + } else {
1.840 + gclog_or_tty->print("[%d %s-%s: "SIZE_FORMAT"K("SIZE_FORMAT"K)]",
1.841 + level(), short_name(), s, used() / K, capacity() / K);
1.842 + }
1.843 + }
1.844 + if (Verbose) {
1.845 + gclog_or_tty->print(" "SIZE_FORMAT"("SIZE_FORMAT")",
1.846 + gch->used(), gch->capacity());
1.847 + } else {
1.848 + gclog_or_tty->print(" "SIZE_FORMAT"K("SIZE_FORMAT"K)",
1.849 + gch->used() / K, gch->capacity() / K);
1.850 + }
1.851 +}
1.852 +
1.853 +size_t
1.854 +ConcurrentMarkSweepGeneration::contiguous_available() const {
1.855 + // dld proposes an improvement in precision here. If the committed
1.856 + // part of the space ends in a free block we should add that to
1.857 + // uncommitted size in the calculation below. Will make this
1.858 + // change later, staying with the approximation below for the
1.859 + // time being. -- ysr.
1.860 + return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
1.861 +}
1.862 +
1.863 +size_t
1.864 +ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
1.865 + return _cmsSpace->max_alloc_in_words() * HeapWordSize;
1.866 +}
1.867 +
1.868 +size_t ConcurrentMarkSweepGeneration::max_available() const {
1.869 + return free() + _virtual_space.uncommitted_size();
1.870 +}
1.871 +
1.872 +bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
1.873 + size_t available = max_available();
1.874 + size_t av_promo = (size_t)gc_stats()->avg_promoted()->padded_average();
1.875 + bool res = (available >= av_promo) || (available >= max_promotion_in_bytes);
1.876 + if (Verbose && PrintGCDetails) {
1.877 + gclog_or_tty->print_cr(
1.878 + "CMS: promo attempt is%s safe: available("SIZE_FORMAT") %s av_promo("SIZE_FORMAT"),"
1.879 + "max_promo("SIZE_FORMAT")",
1.880 + res? "":" not", available, res? ">=":"<",
1.881 + av_promo, max_promotion_in_bytes);
1.882 + }
1.883 + return res;
1.884 +}
1.885 +
1.886 +// At a promotion failure dump information on block layout in heap
1.887 +// (cms old generation).
1.888 +void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {
1.889 + if (CMSDumpAtPromotionFailure) {
1.890 + cmsSpace()->dump_at_safepoint_with_locks(collector(), gclog_or_tty);
1.891 + }
1.892 +}
1.893 +
1.894 +CompactibleSpace*
1.895 +ConcurrentMarkSweepGeneration::first_compaction_space() const {
1.896 + return _cmsSpace;
1.897 +}
1.898 +
1.899 +void ConcurrentMarkSweepGeneration::reset_after_compaction() {
1.900 + // Clear the promotion information. These pointers can be adjusted
1.901 + // along with all the other pointers into the heap but
1.902 + // compaction is expected to be a rare event with
1.903 + // a heap using cms so don't do it without seeing the need.
1.904 + if (CollectedHeap::use_parallel_gc_threads()) {
1.905 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.906 + _par_gc_thread_states[i]->promo.reset();
1.907 + }
1.908 + }
1.909 +}
1.910 +
1.911 +void ConcurrentMarkSweepGeneration::space_iterate(SpaceClosure* blk, bool usedOnly) {
1.912 + blk->do_space(_cmsSpace);
1.913 +}
1.914 +
1.915 +void ConcurrentMarkSweepGeneration::compute_new_size() {
1.916 + assert_locked_or_safepoint(Heap_lock);
1.917 +
1.918 + // If incremental collection failed, we just want to expand
1.919 + // to the limit.
1.920 + if (incremental_collection_failed()) {
1.921 + clear_incremental_collection_failed();
1.922 + grow_to_reserved();
1.923 + return;
1.924 + }
1.925 +
1.926 + // The heap has been compacted but not reset yet.
1.927 + // Any metric such as free() or used() will be incorrect.
1.928 +
1.929 + CardGeneration::compute_new_size();
1.930 +
1.931 + // Reset again after a possible resizing
1.932 + if (did_compact()) {
1.933 + cmsSpace()->reset_after_compaction();
1.934 + }
1.935 +}
1.936 +
1.937 +void ConcurrentMarkSweepGeneration::compute_new_size_free_list() {
1.938 + assert_locked_or_safepoint(Heap_lock);
1.939 +
1.940 + // If incremental collection failed, we just want to expand
1.941 + // to the limit.
1.942 + if (incremental_collection_failed()) {
1.943 + clear_incremental_collection_failed();
1.944 + grow_to_reserved();
1.945 + return;
1.946 + }
1.947 +
1.948 + double free_percentage = ((double) free()) / capacity();
1.949 + double desired_free_percentage = (double) MinHeapFreeRatio / 100;
1.950 + double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
1.951 +
1.952 + // compute expansion delta needed for reaching desired free percentage
1.953 + if (free_percentage < desired_free_percentage) {
1.954 + size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
1.955 + assert(desired_capacity >= capacity(), "invalid expansion size");
1.956 + size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
1.957 + if (PrintGCDetails && Verbose) {
1.958 + size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
1.959 + gclog_or_tty->print_cr("\nFrom compute_new_size: ");
1.960 + gclog_or_tty->print_cr(" Free fraction %f", free_percentage);
1.961 + gclog_or_tty->print_cr(" Desired free fraction %f",
1.962 + desired_free_percentage);
1.963 + gclog_or_tty->print_cr(" Maximum free fraction %f",
1.964 + maximum_free_percentage);
1.965 + gclog_or_tty->print_cr(" Capactiy "SIZE_FORMAT, capacity()/1000);
1.966 + gclog_or_tty->print_cr(" Desired capacity "SIZE_FORMAT,
1.967 + desired_capacity/1000);
1.968 + int prev_level = level() - 1;
1.969 + if (prev_level >= 0) {
1.970 + size_t prev_size = 0;
1.971 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.972 + Generation* prev_gen = gch->_gens[prev_level];
1.973 + prev_size = prev_gen->capacity();
1.974 + gclog_or_tty->print_cr(" Younger gen size "SIZE_FORMAT,
1.975 + prev_size/1000);
1.976 + }
1.977 + gclog_or_tty->print_cr(" unsafe_max_alloc_nogc "SIZE_FORMAT,
1.978 + unsafe_max_alloc_nogc()/1000);
1.979 + gclog_or_tty->print_cr(" contiguous available "SIZE_FORMAT,
1.980 + contiguous_available()/1000);
1.981 + gclog_or_tty->print_cr(" Expand by "SIZE_FORMAT" (bytes)",
1.982 + expand_bytes);
1.983 + }
1.984 + // safe if expansion fails
1.985 + expand(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
1.986 + if (PrintGCDetails && Verbose) {
1.987 + gclog_or_tty->print_cr(" Expanded free fraction %f",
1.988 + ((double) free()) / capacity());
1.989 + }
1.990 + } else {
1.991 + size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
1.992 + assert(desired_capacity <= capacity(), "invalid expansion size");
1.993 + size_t shrink_bytes = capacity() - desired_capacity;
1.994 + // Don't shrink unless the delta is greater than the minimum shrink we want
1.995 + if (shrink_bytes >= MinHeapDeltaBytes) {
1.996 + shrink_free_list_by(shrink_bytes);
1.997 + }
1.998 + }
1.999 +}
1.1000 +
1.1001 +Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
1.1002 + return cmsSpace()->freelistLock();
1.1003 +}
1.1004 +
1.1005 +HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size,
1.1006 + bool tlab) {
1.1007 + CMSSynchronousYieldRequest yr;
1.1008 + MutexLockerEx x(freelistLock(),
1.1009 + Mutex::_no_safepoint_check_flag);
1.1010 + return have_lock_and_allocate(size, tlab);
1.1011 +}
1.1012 +
1.1013 +HeapWord* ConcurrentMarkSweepGeneration::have_lock_and_allocate(size_t size,
1.1014 + bool tlab /* ignored */) {
1.1015 + assert_lock_strong(freelistLock());
1.1016 + size_t adjustedSize = CompactibleFreeListSpace::adjustObjectSize(size);
1.1017 + HeapWord* res = cmsSpace()->allocate(adjustedSize);
1.1018 + // Allocate the object live (grey) if the background collector has
1.1019 + // started marking. This is necessary because the marker may
1.1020 + // have passed this address and consequently this object will
1.1021 + // not otherwise be greyed and would be incorrectly swept up.
1.1022 + // Note that if this object contains references, the writing
1.1023 + // of those references will dirty the card containing this object
1.1024 + // allowing the object to be blackened (and its references scanned)
1.1025 + // either during a preclean phase or at the final checkpoint.
1.1026 + if (res != NULL) {
1.1027 + // We may block here with an uninitialized object with
1.1028 + // its mark-bit or P-bits not yet set. Such objects need
1.1029 + // to be safely navigable by block_start().
1.1030 + assert(oop(res)->klass_or_null() == NULL, "Object should be uninitialized here.");
1.1031 + assert(!((FreeChunk*)res)->is_free(), "Error, block will look free but show wrong size");
1.1032 + collector()->direct_allocated(res, adjustedSize);
1.1033 + _direct_allocated_words += adjustedSize;
1.1034 + // allocation counters
1.1035 + NOT_PRODUCT(
1.1036 + _numObjectsAllocated++;
1.1037 + _numWordsAllocated += (int)adjustedSize;
1.1038 + )
1.1039 + }
1.1040 + return res;
1.1041 +}
1.1042 +
1.1043 +// In the case of direct allocation by mutators in a generation that
1.1044 +// is being concurrently collected, the object must be allocated
1.1045 +// live (grey) if the background collector has started marking.
1.1046 +// This is necessary because the marker may
1.1047 +// have passed this address and consequently this object will
1.1048 +// not otherwise be greyed and would be incorrectly swept up.
1.1049 +// Note that if this object contains references, the writing
1.1050 +// of those references will dirty the card containing this object
1.1051 +// allowing the object to be blackened (and its references scanned)
1.1052 +// either during a preclean phase or at the final checkpoint.
1.1053 +void CMSCollector::direct_allocated(HeapWord* start, size_t size) {
1.1054 + assert(_markBitMap.covers(start, size), "Out of bounds");
1.1055 + if (_collectorState >= Marking) {
1.1056 + MutexLockerEx y(_markBitMap.lock(),
1.1057 + Mutex::_no_safepoint_check_flag);
1.1058 + // [see comments preceding SweepClosure::do_blk() below for details]
1.1059 + //
1.1060 + // Can the P-bits be deleted now? JJJ
1.1061 + //
1.1062 + // 1. need to mark the object as live so it isn't collected
1.1063 + // 2. need to mark the 2nd bit to indicate the object may be uninitialized
1.1064 + // 3. need to mark the end of the object so marking, precleaning or sweeping
1.1065 + // can skip over uninitialized or unparsable objects. An allocated
1.1066 + // object is considered uninitialized for our purposes as long as
1.1067 + // its klass word is NULL. All old gen objects are parsable
1.1068 + // as soon as they are initialized.)
1.1069 + _markBitMap.mark(start); // object is live
1.1070 + _markBitMap.mark(start + 1); // object is potentially uninitialized?
1.1071 + _markBitMap.mark(start + size - 1);
1.1072 + // mark end of object
1.1073 + }
1.1074 + // check that oop looks uninitialized
1.1075 + assert(oop(start)->klass_or_null() == NULL, "_klass should be NULL");
1.1076 +}
1.1077 +
1.1078 +void CMSCollector::promoted(bool par, HeapWord* start,
1.1079 + bool is_obj_array, size_t obj_size) {
1.1080 + assert(_markBitMap.covers(start), "Out of bounds");
1.1081 + // See comment in direct_allocated() about when objects should
1.1082 + // be allocated live.
1.1083 + if (_collectorState >= Marking) {
1.1084 + // we already hold the marking bit map lock, taken in
1.1085 + // the prologue
1.1086 + if (par) {
1.1087 + _markBitMap.par_mark(start);
1.1088 + } else {
1.1089 + _markBitMap.mark(start);
1.1090 + }
1.1091 + // We don't need to mark the object as uninitialized (as
1.1092 + // in direct_allocated above) because this is being done with the
1.1093 + // world stopped and the object will be initialized by the
1.1094 + // time the marking, precleaning or sweeping get to look at it.
1.1095 + // But see the code for copying objects into the CMS generation,
1.1096 + // where we need to ensure that concurrent readers of the
1.1097 + // block offset table are able to safely navigate a block that
1.1098 + // is in flux from being free to being allocated (and in
1.1099 + // transition while being copied into) and subsequently
1.1100 + // becoming a bona-fide object when the copy/promotion is complete.
1.1101 + assert(SafepointSynchronize::is_at_safepoint(),
1.1102 + "expect promotion only at safepoints");
1.1103 +
1.1104 + if (_collectorState < Sweeping) {
1.1105 + // Mark the appropriate cards in the modUnionTable, so that
1.1106 + // this object gets scanned before the sweep. If this is
1.1107 + // not done, CMS generation references in the object might
1.1108 + // not get marked.
1.1109 + // For the case of arrays, which are otherwise precisely
1.1110 + // marked, we need to dirty the entire array, not just its head.
1.1111 + if (is_obj_array) {
1.1112 + // The [par_]mark_range() method expects mr.end() below to
1.1113 + // be aligned to the granularity of a bit's representation
1.1114 + // in the heap. In the case of the MUT below, that's a
1.1115 + // card size.
1.1116 + MemRegion mr(start,
1.1117 + (HeapWord*)round_to((intptr_t)(start + obj_size),
1.1118 + CardTableModRefBS::card_size /* bytes */));
1.1119 + if (par) {
1.1120 + _modUnionTable.par_mark_range(mr);
1.1121 + } else {
1.1122 + _modUnionTable.mark_range(mr);
1.1123 + }
1.1124 + } else { // not an obj array; we can just mark the head
1.1125 + if (par) {
1.1126 + _modUnionTable.par_mark(start);
1.1127 + } else {
1.1128 + _modUnionTable.mark(start);
1.1129 + }
1.1130 + }
1.1131 + }
1.1132 + }
1.1133 +}
1.1134 +
1.1135 +static inline size_t percent_of_space(Space* space, HeapWord* addr)
1.1136 +{
1.1137 + size_t delta = pointer_delta(addr, space->bottom());
1.1138 + return (size_t)(delta * 100.0 / (space->capacity() / HeapWordSize));
1.1139 +}
1.1140 +
1.1141 +void CMSCollector::icms_update_allocation_limits()
1.1142 +{
1.1143 + Generation* gen0 = GenCollectedHeap::heap()->get_gen(0);
1.1144 + EdenSpace* eden = gen0->as_DefNewGeneration()->eden();
1.1145 +
1.1146 + const unsigned int duty_cycle = stats().icms_update_duty_cycle();
1.1147 + if (CMSTraceIncrementalPacing) {
1.1148 + stats().print();
1.1149 + }
1.1150 +
1.1151 + assert(duty_cycle <= 100, "invalid duty cycle");
1.1152 + if (duty_cycle != 0) {
1.1153 + // The duty_cycle is a percentage between 0 and 100; convert to words and
1.1154 + // then compute the offset from the endpoints of the space.
1.1155 + size_t free_words = eden->free() / HeapWordSize;
1.1156 + double free_words_dbl = (double)free_words;
1.1157 + size_t duty_cycle_words = (size_t)(free_words_dbl * duty_cycle / 100.0);
1.1158 + size_t offset_words = (free_words - duty_cycle_words) / 2;
1.1159 +
1.1160 + _icms_start_limit = eden->top() + offset_words;
1.1161 + _icms_stop_limit = eden->end() - offset_words;
1.1162 +
1.1163 + // The limits may be adjusted (shifted to the right) by
1.1164 + // CMSIncrementalOffset, to allow the application more mutator time after a
1.1165 + // young gen gc (when all mutators were stopped) and before CMS starts and
1.1166 + // takes away one or more cpus.
1.1167 + if (CMSIncrementalOffset != 0) {
1.1168 + double adjustment_dbl = free_words_dbl * CMSIncrementalOffset / 100.0;
1.1169 + size_t adjustment = (size_t)adjustment_dbl;
1.1170 + HeapWord* tmp_stop = _icms_stop_limit + adjustment;
1.1171 + if (tmp_stop > _icms_stop_limit && tmp_stop < eden->end()) {
1.1172 + _icms_start_limit += adjustment;
1.1173 + _icms_stop_limit = tmp_stop;
1.1174 + }
1.1175 + }
1.1176 + }
1.1177 + if (duty_cycle == 0 || (_icms_start_limit == _icms_stop_limit)) {
1.1178 + _icms_start_limit = _icms_stop_limit = eden->end();
1.1179 + }
1.1180 +
1.1181 + // Install the new start limit.
1.1182 + eden->set_soft_end(_icms_start_limit);
1.1183 +
1.1184 + if (CMSTraceIncrementalMode) {
1.1185 + gclog_or_tty->print(" icms alloc limits: "
1.1186 + PTR_FORMAT "," PTR_FORMAT
1.1187 + " (" SIZE_FORMAT "%%," SIZE_FORMAT "%%) ",
1.1188 + p2i(_icms_start_limit), p2i(_icms_stop_limit),
1.1189 + percent_of_space(eden, _icms_start_limit),
1.1190 + percent_of_space(eden, _icms_stop_limit));
1.1191 + if (Verbose) {
1.1192 + gclog_or_tty->print("eden: ");
1.1193 + eden->print_on(gclog_or_tty);
1.1194 + }
1.1195 + }
1.1196 +}
1.1197 +
1.1198 +// Any changes here should try to maintain the invariant
1.1199 +// that if this method is called with _icms_start_limit
1.1200 +// and _icms_stop_limit both NULL, then it should return NULL
1.1201 +// and not notify the icms thread.
1.1202 +HeapWord*
1.1203 +CMSCollector::allocation_limit_reached(Space* space, HeapWord* top,
1.1204 + size_t word_size)
1.1205 +{
1.1206 + // A start_limit equal to end() means the duty cycle is 0, so treat that as a
1.1207 + // nop.
1.1208 + if (CMSIncrementalMode && _icms_start_limit != space->end()) {
1.1209 + if (top <= _icms_start_limit) {
1.1210 + if (CMSTraceIncrementalMode) {
1.1211 + space->print_on(gclog_or_tty);
1.1212 + gclog_or_tty->stamp();
1.1213 + gclog_or_tty->print_cr(" start limit top=" PTR_FORMAT
1.1214 + ", new limit=" PTR_FORMAT
1.1215 + " (" SIZE_FORMAT "%%)",
1.1216 + p2i(top), p2i(_icms_stop_limit),
1.1217 + percent_of_space(space, _icms_stop_limit));
1.1218 + }
1.1219 + ConcurrentMarkSweepThread::start_icms();
1.1220 + assert(top < _icms_stop_limit, "Tautology");
1.1221 + if (word_size < pointer_delta(_icms_stop_limit, top)) {
1.1222 + return _icms_stop_limit;
1.1223 + }
1.1224 +
1.1225 + // The allocation will cross both the _start and _stop limits, so do the
1.1226 + // stop notification also and return end().
1.1227 + if (CMSTraceIncrementalMode) {
1.1228 + space->print_on(gclog_or_tty);
1.1229 + gclog_or_tty->stamp();
1.1230 + gclog_or_tty->print_cr(" +stop limit top=" PTR_FORMAT
1.1231 + ", new limit=" PTR_FORMAT
1.1232 + " (" SIZE_FORMAT "%%)",
1.1233 + p2i(top), p2i(space->end()),
1.1234 + percent_of_space(space, space->end()));
1.1235 + }
1.1236 + ConcurrentMarkSweepThread::stop_icms();
1.1237 + return space->end();
1.1238 + }
1.1239 +
1.1240 + if (top <= _icms_stop_limit) {
1.1241 + if (CMSTraceIncrementalMode) {
1.1242 + space->print_on(gclog_or_tty);
1.1243 + gclog_or_tty->stamp();
1.1244 + gclog_or_tty->print_cr(" stop limit top=" PTR_FORMAT
1.1245 + ", new limit=" PTR_FORMAT
1.1246 + " (" SIZE_FORMAT "%%)",
1.1247 + top, space->end(),
1.1248 + percent_of_space(space, space->end()));
1.1249 + }
1.1250 + ConcurrentMarkSweepThread::stop_icms();
1.1251 + return space->end();
1.1252 + }
1.1253 +
1.1254 + if (CMSTraceIncrementalMode) {
1.1255 + space->print_on(gclog_or_tty);
1.1256 + gclog_or_tty->stamp();
1.1257 + gclog_or_tty->print_cr(" end limit top=" PTR_FORMAT
1.1258 + ", new limit=" PTR_FORMAT,
1.1259 + top, NULL);
1.1260 + }
1.1261 + }
1.1262 +
1.1263 + return NULL;
1.1264 +}
1.1265 +
1.1266 +oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) {
1.1267 + assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");
1.1268 + // allocate, copy and if necessary update promoinfo --
1.1269 + // delegate to underlying space.
1.1270 + assert_lock_strong(freelistLock());
1.1271 +
1.1272 +#ifndef PRODUCT
1.1273 + if (Universe::heap()->promotion_should_fail()) {
1.1274 + return NULL;
1.1275 + }
1.1276 +#endif // #ifndef PRODUCT
1.1277 +
1.1278 + oop res = _cmsSpace->promote(obj, obj_size);
1.1279 + if (res == NULL) {
1.1280 + // expand and retry
1.1281 + size_t s = _cmsSpace->expansionSpaceRequired(obj_size); // HeapWords
1.1282 + expand(s*HeapWordSize, MinHeapDeltaBytes,
1.1283 + CMSExpansionCause::_satisfy_promotion);
1.1284 + // Since there's currently no next generation, we don't try to promote
1.1285 + // into a more senior generation.
1.1286 + assert(next_gen() == NULL, "assumption, based upon which no attempt "
1.1287 + "is made to pass on a possibly failing "
1.1288 + "promotion to next generation");
1.1289 + res = _cmsSpace->promote(obj, obj_size);
1.1290 + }
1.1291 + if (res != NULL) {
1.1292 + // See comment in allocate() about when objects should
1.1293 + // be allocated live.
1.1294 + assert(obj->is_oop(), "Will dereference klass pointer below");
1.1295 + collector()->promoted(false, // Not parallel
1.1296 + (HeapWord*)res, obj->is_objArray(), obj_size);
1.1297 + // promotion counters
1.1298 + NOT_PRODUCT(
1.1299 + _numObjectsPromoted++;
1.1300 + _numWordsPromoted +=
1.1301 + (int)(CompactibleFreeListSpace::adjustObjectSize(obj->size()));
1.1302 + )
1.1303 + }
1.1304 + return res;
1.1305 +}
1.1306 +
1.1307 +
1.1308 +HeapWord*
1.1309 +ConcurrentMarkSweepGeneration::allocation_limit_reached(Space* space,
1.1310 + HeapWord* top,
1.1311 + size_t word_sz)
1.1312 +{
1.1313 + return collector()->allocation_limit_reached(space, top, word_sz);
1.1314 +}
1.1315 +
1.1316 +// IMPORTANT: Notes on object size recognition in CMS.
1.1317 +// ---------------------------------------------------
1.1318 +// A block of storage in the CMS generation is always in
1.1319 +// one of three states. A free block (FREE), an allocated
1.1320 +// object (OBJECT) whose size() method reports the correct size,
1.1321 +// and an intermediate state (TRANSIENT) in which its size cannot
1.1322 +// be accurately determined.
1.1323 +// STATE IDENTIFICATION: (32 bit and 64 bit w/o COOPS)
1.1324 +// -----------------------------------------------------
1.1325 +// FREE: klass_word & 1 == 1; mark_word holds block size
1.1326 +//
1.1327 +// OBJECT: klass_word installed; klass_word != 0 && klass_word & 1 == 0;
1.1328 +// obj->size() computes correct size
1.1329 +//
1.1330 +// TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT
1.1331 +//
1.1332 +// STATE IDENTIFICATION: (64 bit+COOPS)
1.1333 +// ------------------------------------
1.1334 +// FREE: mark_word & CMS_FREE_BIT == 1; mark_word & ~CMS_FREE_BIT gives block_size
1.1335 +//
1.1336 +// OBJECT: klass_word installed; klass_word != 0;
1.1337 +// obj->size() computes correct size
1.1338 +//
1.1339 +// TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT
1.1340 +//
1.1341 +//
1.1342 +// STATE TRANSITION DIAGRAM
1.1343 +//
1.1344 +// mut / parnew mut / parnew
1.1345 +// FREE --------------------> TRANSIENT ---------------------> OBJECT --|
1.1346 +// ^ |
1.1347 +// |------------------------ DEAD <------------------------------------|
1.1348 +// sweep mut
1.1349 +//
1.1350 +// While a block is in TRANSIENT state its size cannot be determined
1.1351 +// so readers will either need to come back later or stall until
1.1352 +// the size can be determined. Note that for the case of direct
1.1353 +// allocation, P-bits, when available, may be used to determine the
1.1354 +// size of an object that may not yet have been initialized.
1.1355 +
1.1356 +// Things to support parallel young-gen collection.
1.1357 +oop
1.1358 +ConcurrentMarkSweepGeneration::par_promote(int thread_num,
1.1359 + oop old, markOop m,
1.1360 + size_t word_sz) {
1.1361 +#ifndef PRODUCT
1.1362 + if (Universe::heap()->promotion_should_fail()) {
1.1363 + return NULL;
1.1364 + }
1.1365 +#endif // #ifndef PRODUCT
1.1366 +
1.1367 + CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1.1368 + PromotionInfo* promoInfo = &ps->promo;
1.1369 + // if we are tracking promotions, then first ensure space for
1.1370 + // promotion (including spooling space for saving header if necessary).
1.1371 + // then allocate and copy, then track promoted info if needed.
1.1372 + // When tracking (see PromotionInfo::track()), the mark word may
1.1373 + // be displaced and in this case restoration of the mark word
1.1374 + // occurs in the (oop_since_save_marks_)iterate phase.
1.1375 + if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {
1.1376 + // Out of space for allocating spooling buffers;
1.1377 + // try expanding and allocating spooling buffers.
1.1378 + if (!expand_and_ensure_spooling_space(promoInfo)) {
1.1379 + return NULL;
1.1380 + }
1.1381 + }
1.1382 + assert(promoInfo->has_spooling_space(), "Control point invariant");
1.1383 + const size_t alloc_sz = CompactibleFreeListSpace::adjustObjectSize(word_sz);
1.1384 + HeapWord* obj_ptr = ps->lab.alloc(alloc_sz);
1.1385 + if (obj_ptr == NULL) {
1.1386 + obj_ptr = expand_and_par_lab_allocate(ps, alloc_sz);
1.1387 + if (obj_ptr == NULL) {
1.1388 + return NULL;
1.1389 + }
1.1390 + }
1.1391 + oop obj = oop(obj_ptr);
1.1392 + OrderAccess::storestore();
1.1393 + assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
1.1394 + assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");
1.1395 + // IMPORTANT: See note on object initialization for CMS above.
1.1396 + // Otherwise, copy the object. Here we must be careful to insert the
1.1397 + // klass pointer last, since this marks the block as an allocated object.
1.1398 + // Except with compressed oops it's the mark word.
1.1399 + HeapWord* old_ptr = (HeapWord*)old;
1.1400 + // Restore the mark word copied above.
1.1401 + obj->set_mark(m);
1.1402 + assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
1.1403 + assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");
1.1404 + OrderAccess::storestore();
1.1405 +
1.1406 + if (UseCompressedClassPointers) {
1.1407 + // Copy gap missed by (aligned) header size calculation below
1.1408 + obj->set_klass_gap(old->klass_gap());
1.1409 + }
1.1410 + if (word_sz > (size_t)oopDesc::header_size()) {
1.1411 + Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),
1.1412 + obj_ptr + oopDesc::header_size(),
1.1413 + word_sz - oopDesc::header_size());
1.1414 + }
1.1415 +
1.1416 + // Now we can track the promoted object, if necessary. We take care
1.1417 + // to delay the transition from uninitialized to full object
1.1418 + // (i.e., insertion of klass pointer) until after, so that it
1.1419 + // atomically becomes a promoted object.
1.1420 + if (promoInfo->tracking()) {
1.1421 + promoInfo->track((PromotedObject*)obj, old->klass());
1.1422 + }
1.1423 + assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
1.1424 + assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");
1.1425 + assert(old->is_oop(), "Will use and dereference old klass ptr below");
1.1426 +
1.1427 + // Finally, install the klass pointer (this should be volatile).
1.1428 + OrderAccess::storestore();
1.1429 + obj->set_klass(old->klass());
1.1430 + // We should now be able to calculate the right size for this object
1.1431 + assert(obj->is_oop() && obj->size() == (int)word_sz, "Error, incorrect size computed for promoted object");
1.1432 +
1.1433 + collector()->promoted(true, // parallel
1.1434 + obj_ptr, old->is_objArray(), word_sz);
1.1435 +
1.1436 + NOT_PRODUCT(
1.1437 + Atomic::inc_ptr(&_numObjectsPromoted);
1.1438 + Atomic::add_ptr(alloc_sz, &_numWordsPromoted);
1.1439 + )
1.1440 +
1.1441 + return obj;
1.1442 +}
1.1443 +
1.1444 +void
1.1445 +ConcurrentMarkSweepGeneration::
1.1446 +par_promote_alloc_undo(int thread_num,
1.1447 + HeapWord* obj, size_t word_sz) {
1.1448 + // CMS does not support promotion undo.
1.1449 + ShouldNotReachHere();
1.1450 +}
1.1451 +
1.1452 +void
1.1453 +ConcurrentMarkSweepGeneration::
1.1454 +par_promote_alloc_done(int thread_num) {
1.1455 + CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1.1456 + ps->lab.retire(thread_num);
1.1457 +}
1.1458 +
1.1459 +void
1.1460 +ConcurrentMarkSweepGeneration::
1.1461 +par_oop_since_save_marks_iterate_done(int thread_num) {
1.1462 + CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1.1463 + ParScanWithoutBarrierClosure* dummy_cl = NULL;
1.1464 + ps->promo.promoted_oops_iterate_nv(dummy_cl);
1.1465 +}
1.1466 +
1.1467 +bool ConcurrentMarkSweepGeneration::should_collect(bool full,
1.1468 + size_t size,
1.1469 + bool tlab)
1.1470 +{
1.1471 + // We allow a STW collection only if a full
1.1472 + // collection was requested.
1.1473 + return full || should_allocate(size, tlab); // FIX ME !!!
1.1474 + // This and promotion failure handling are connected at the
1.1475 + // hip and should be fixed by untying them.
1.1476 +}
1.1477 +
1.1478 +bool CMSCollector::shouldConcurrentCollect() {
1.1479 + if (_full_gc_requested) {
1.1480 + if (Verbose && PrintGCDetails) {
1.1481 + gclog_or_tty->print_cr("CMSCollector: collect because of explicit "
1.1482 + " gc request (or gc_locker)");
1.1483 + }
1.1484 + return true;
1.1485 + }
1.1486 +
1.1487 + // For debugging purposes, change the type of collection.
1.1488 + // If the rotation is not on the concurrent collection
1.1489 + // type, don't start a concurrent collection.
1.1490 + NOT_PRODUCT(
1.1491 + if (RotateCMSCollectionTypes &&
1.1492 + (_cmsGen->debug_collection_type() !=
1.1493 + ConcurrentMarkSweepGeneration::Concurrent_collection_type)) {
1.1494 + assert(_cmsGen->debug_collection_type() !=
1.1495 + ConcurrentMarkSweepGeneration::Unknown_collection_type,
1.1496 + "Bad cms collection type");
1.1497 + return false;
1.1498 + }
1.1499 + )
1.1500 +
1.1501 + FreelistLocker x(this);
1.1502 + // ------------------------------------------------------------------
1.1503 + // Print out lots of information which affects the initiation of
1.1504 + // a collection.
1.1505 + if (PrintCMSInitiationStatistics && stats().valid()) {
1.1506 + gclog_or_tty->print("CMSCollector shouldConcurrentCollect: ");
1.1507 + gclog_or_tty->stamp();
1.1508 + gclog_or_tty->cr();
1.1509 + stats().print_on(gclog_or_tty);
1.1510 + gclog_or_tty->print_cr("time_until_cms_gen_full %3.7f",
1.1511 + stats().time_until_cms_gen_full());
1.1512 + gclog_or_tty->print_cr("free="SIZE_FORMAT, _cmsGen->free());
1.1513 + gclog_or_tty->print_cr("contiguous_available="SIZE_FORMAT,
1.1514 + _cmsGen->contiguous_available());
1.1515 + gclog_or_tty->print_cr("promotion_rate=%g", stats().promotion_rate());
1.1516 + gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate());
1.1517 + gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy());
1.1518 + gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy());
1.1519 + gclog_or_tty->print_cr("metadata initialized %d",
1.1520 + MetaspaceGC::should_concurrent_collect());
1.1521 + }
1.1522 + // ------------------------------------------------------------------
1.1523 +
1.1524 + // If the estimated time to complete a cms collection (cms_duration())
1.1525 + // is less than the estimated time remaining until the cms generation
1.1526 + // is full, start a collection.
1.1527 + if (!UseCMSInitiatingOccupancyOnly) {
1.1528 + if (stats().valid()) {
1.1529 + if (stats().time_until_cms_start() == 0.0) {
1.1530 + return true;
1.1531 + }
1.1532 + } else {
1.1533 + // We want to conservatively collect somewhat early in order
1.1534 + // to try and "bootstrap" our CMS/promotion statistics;
1.1535 + // this branch will not fire after the first successful CMS
1.1536 + // collection because the stats should then be valid.
1.1537 + if (_cmsGen->occupancy() >= _bootstrap_occupancy) {
1.1538 + if (Verbose && PrintGCDetails) {
1.1539 + gclog_or_tty->print_cr(
1.1540 + " CMSCollector: collect for bootstrapping statistics:"
1.1541 + " occupancy = %f, boot occupancy = %f", _cmsGen->occupancy(),
1.1542 + _bootstrap_occupancy);
1.1543 + }
1.1544 + return true;
1.1545 + }
1.1546 + }
1.1547 + }
1.1548 +
1.1549 + // Otherwise, we start a collection cycle if
1.1550 + // old gen want a collection cycle started. Each may use
1.1551 + // an appropriate criterion for making this decision.
1.1552 + // XXX We need to make sure that the gen expansion
1.1553 + // criterion dovetails well with this. XXX NEED TO FIX THIS
1.1554 + if (_cmsGen->should_concurrent_collect()) {
1.1555 + if (Verbose && PrintGCDetails) {
1.1556 + gclog_or_tty->print_cr("CMS old gen initiated");
1.1557 + }
1.1558 + return true;
1.1559 + }
1.1560 +
1.1561 + // We start a collection if we believe an incremental collection may fail;
1.1562 + // this is not likely to be productive in practice because it's probably too
1.1563 + // late anyway.
1.1564 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.1565 + assert(gch->collector_policy()->is_two_generation_policy(),
1.1566 + "You may want to check the correctness of the following");
1.1567 + if (gch->incremental_collection_will_fail(true /* consult_young */)) {
1.1568 + if (Verbose && PrintGCDetails) {
1.1569 + gclog_or_tty->print("CMSCollector: collect because incremental collection will fail ");
1.1570 + }
1.1571 + return true;
1.1572 + }
1.1573 +
1.1574 + if (MetaspaceGC::should_concurrent_collect()) {
1.1575 + if (Verbose && PrintGCDetails) {
1.1576 + gclog_or_tty->print("CMSCollector: collect for metadata allocation ");
1.1577 + }
1.1578 + return true;
1.1579 + }
1.1580 +
1.1581 + return false;
1.1582 +}
1.1583 +
1.1584 +void CMSCollector::set_did_compact(bool v) { _cmsGen->set_did_compact(v); }
1.1585 +
1.1586 +// Clear _expansion_cause fields of constituent generations
1.1587 +void CMSCollector::clear_expansion_cause() {
1.1588 + _cmsGen->clear_expansion_cause();
1.1589 +}
1.1590 +
1.1591 +// We should be conservative in starting a collection cycle. To
1.1592 +// start too eagerly runs the risk of collecting too often in the
1.1593 +// extreme. To collect too rarely falls back on full collections,
1.1594 +// which works, even if not optimum in terms of concurrent work.
1.1595 +// As a work around for too eagerly collecting, use the flag
1.1596 +// UseCMSInitiatingOccupancyOnly. This also has the advantage of
1.1597 +// giving the user an easily understandable way of controlling the
1.1598 +// collections.
1.1599 +// We want to start a new collection cycle if any of the following
1.1600 +// conditions hold:
1.1601 +// . our current occupancy exceeds the configured initiating occupancy
1.1602 +// for this generation, or
1.1603 +// . we recently needed to expand this space and have not, since that
1.1604 +// expansion, done a collection of this generation, or
1.1605 +// . the underlying space believes that it may be a good idea to initiate
1.1606 +// a concurrent collection (this may be based on criteria such as the
1.1607 +// following: the space uses linear allocation and linear allocation is
1.1608 +// going to fail, or there is believed to be excessive fragmentation in
1.1609 +// the generation, etc... or ...
1.1610 +// [.(currently done by CMSCollector::shouldConcurrentCollect() only for
1.1611 +// the case of the old generation; see CR 6543076):
1.1612 +// we may be approaching a point at which allocation requests may fail because
1.1613 +// we will be out of sufficient free space given allocation rate estimates.]
1.1614 +bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const {
1.1615 +
1.1616 + assert_lock_strong(freelistLock());
1.1617 + if (occupancy() > initiating_occupancy()) {
1.1618 + if (PrintGCDetails && Verbose) {
1.1619 + gclog_or_tty->print(" %s: collect because of occupancy %f / %f ",
1.1620 + short_name(), occupancy(), initiating_occupancy());
1.1621 + }
1.1622 + return true;
1.1623 + }
1.1624 + if (UseCMSInitiatingOccupancyOnly) {
1.1625 + return false;
1.1626 + }
1.1627 + if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) {
1.1628 + if (PrintGCDetails && Verbose) {
1.1629 + gclog_or_tty->print(" %s: collect because expanded for allocation ",
1.1630 + short_name());
1.1631 + }
1.1632 + return true;
1.1633 + }
1.1634 + if (_cmsSpace->should_concurrent_collect()) {
1.1635 + if (PrintGCDetails && Verbose) {
1.1636 + gclog_or_tty->print(" %s: collect because cmsSpace says so ",
1.1637 + short_name());
1.1638 + }
1.1639 + return true;
1.1640 + }
1.1641 + return false;
1.1642 +}
1.1643 +
1.1644 +void ConcurrentMarkSweepGeneration::collect(bool full,
1.1645 + bool clear_all_soft_refs,
1.1646 + size_t size,
1.1647 + bool tlab)
1.1648 +{
1.1649 + collector()->collect(full, clear_all_soft_refs, size, tlab);
1.1650 +}
1.1651 +
1.1652 +void CMSCollector::collect(bool full,
1.1653 + bool clear_all_soft_refs,
1.1654 + size_t size,
1.1655 + bool tlab)
1.1656 +{
1.1657 + if (!UseCMSCollectionPassing && _collectorState > Idling) {
1.1658 + // For debugging purposes skip the collection if the state
1.1659 + // is not currently idle
1.1660 + if (TraceCMSState) {
1.1661 + gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " skipped full:%d CMS state %d",
1.1662 + Thread::current(), full, _collectorState);
1.1663 + }
1.1664 + return;
1.1665 + }
1.1666 +
1.1667 + // The following "if" branch is present for defensive reasons.
1.1668 + // In the current uses of this interface, it can be replaced with:
1.1669 + // assert(!GC_locker.is_active(), "Can't be called otherwise");
1.1670 + // But I am not placing that assert here to allow future
1.1671 + // generality in invoking this interface.
1.1672 + if (GC_locker::is_active()) {
1.1673 + // A consistency test for GC_locker
1.1674 + assert(GC_locker::needs_gc(), "Should have been set already");
1.1675 + // Skip this foreground collection, instead
1.1676 + // expanding the heap if necessary.
1.1677 + // Need the free list locks for the call to free() in compute_new_size()
1.1678 + compute_new_size();
1.1679 + return;
1.1680 + }
1.1681 + acquire_control_and_collect(full, clear_all_soft_refs);
1.1682 + _full_gcs_since_conc_gc++;
1.1683 +}
1.1684 +
1.1685 +void CMSCollector::request_full_gc(unsigned int full_gc_count, GCCause::Cause cause) {
1.1686 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.1687 + unsigned int gc_count = gch->total_full_collections();
1.1688 + if (gc_count == full_gc_count) {
1.1689 + MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag);
1.1690 + _full_gc_requested = true;
1.1691 + _full_gc_cause = cause;
1.1692 + CGC_lock->notify(); // nudge CMS thread
1.1693 + } else {
1.1694 + assert(gc_count > full_gc_count, "Error: causal loop");
1.1695 + }
1.1696 +}
1.1697 +
1.1698 +bool CMSCollector::is_external_interruption() {
1.1699 + GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause();
1.1700 + return GCCause::is_user_requested_gc(cause) ||
1.1701 + GCCause::is_serviceability_requested_gc(cause);
1.1702 +}
1.1703 +
1.1704 +void CMSCollector::report_concurrent_mode_interruption() {
1.1705 + if (is_external_interruption()) {
1.1706 + if (PrintGCDetails) {
1.1707 + gclog_or_tty->print(" (concurrent mode interrupted)");
1.1708 + }
1.1709 + } else {
1.1710 + if (PrintGCDetails) {
1.1711 + gclog_or_tty->print(" (concurrent mode failure)");
1.1712 + }
1.1713 + _gc_tracer_cm->report_concurrent_mode_failure();
1.1714 + }
1.1715 +}
1.1716 +
1.1717 +
1.1718 +// The foreground and background collectors need to coordinate in order
1.1719 +// to make sure that they do not mutually interfere with CMS collections.
1.1720 +// When a background collection is active,
1.1721 +// the foreground collector may need to take over (preempt) and
1.1722 +// synchronously complete an ongoing collection. Depending on the
1.1723 +// frequency of the background collections and the heap usage
1.1724 +// of the application, this preemption can be seldom or frequent.
1.1725 +// There are only certain
1.1726 +// points in the background collection that the "collection-baton"
1.1727 +// can be passed to the foreground collector.
1.1728 +//
1.1729 +// The foreground collector will wait for the baton before
1.1730 +// starting any part of the collection. The foreground collector
1.1731 +// will only wait at one location.
1.1732 +//
1.1733 +// The background collector will yield the baton before starting a new
1.1734 +// phase of the collection (e.g., before initial marking, marking from roots,
1.1735 +// precleaning, final re-mark, sweep etc.) This is normally done at the head
1.1736 +// of the loop which switches the phases. The background collector does some
1.1737 +// of the phases (initial mark, final re-mark) with the world stopped.
1.1738 +// Because of locking involved in stopping the world,
1.1739 +// the foreground collector should not block waiting for the background
1.1740 +// collector when it is doing a stop-the-world phase. The background
1.1741 +// collector will yield the baton at an additional point just before
1.1742 +// it enters a stop-the-world phase. Once the world is stopped, the
1.1743 +// background collector checks the phase of the collection. If the
1.1744 +// phase has not changed, it proceeds with the collection. If the
1.1745 +// phase has changed, it skips that phase of the collection. See
1.1746 +// the comments on the use of the Heap_lock in collect_in_background().
1.1747 +//
1.1748 +// Variable used in baton passing.
1.1749 +// _foregroundGCIsActive - Set to true by the foreground collector when
1.1750 +// it wants the baton. The foreground clears it when it has finished
1.1751 +// the collection.
1.1752 +// _foregroundGCShouldWait - Set to true by the background collector
1.1753 +// when it is running. The foreground collector waits while
1.1754 +// _foregroundGCShouldWait is true.
1.1755 +// CGC_lock - monitor used to protect access to the above variables
1.1756 +// and to notify the foreground and background collectors.
1.1757 +// _collectorState - current state of the CMS collection.
1.1758 +//
1.1759 +// The foreground collector
1.1760 +// acquires the CGC_lock
1.1761 +// sets _foregroundGCIsActive
1.1762 +// waits on the CGC_lock for _foregroundGCShouldWait to be false
1.1763 +// various locks acquired in preparation for the collection
1.1764 +// are released so as not to block the background collector
1.1765 +// that is in the midst of a collection
1.1766 +// proceeds with the collection
1.1767 +// clears _foregroundGCIsActive
1.1768 +// returns
1.1769 +//
1.1770 +// The background collector in a loop iterating on the phases of the
1.1771 +// collection
1.1772 +// acquires the CGC_lock
1.1773 +// sets _foregroundGCShouldWait
1.1774 +// if _foregroundGCIsActive is set
1.1775 +// clears _foregroundGCShouldWait, notifies _CGC_lock
1.1776 +// waits on _CGC_lock for _foregroundGCIsActive to become false
1.1777 +// and exits the loop.
1.1778 +// otherwise
1.1779 +// proceed with that phase of the collection
1.1780 +// if the phase is a stop-the-world phase,
1.1781 +// yield the baton once more just before enqueueing
1.1782 +// the stop-world CMS operation (executed by the VM thread).
1.1783 +// returns after all phases of the collection are done
1.1784 +//
1.1785 +
1.1786 +void CMSCollector::acquire_control_and_collect(bool full,
1.1787 + bool clear_all_soft_refs) {
1.1788 + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
1.1789 + assert(!Thread::current()->is_ConcurrentGC_thread(),
1.1790 + "shouldn't try to acquire control from self!");
1.1791 +
1.1792 + // Start the protocol for acquiring control of the
1.1793 + // collection from the background collector (aka CMS thread).
1.1794 + assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1.1795 + "VM thread should have CMS token");
1.1796 + // Remember the possibly interrupted state of an ongoing
1.1797 + // concurrent collection
1.1798 + CollectorState first_state = _collectorState;
1.1799 +
1.1800 + // Signal to a possibly ongoing concurrent collection that
1.1801 + // we want to do a foreground collection.
1.1802 + _foregroundGCIsActive = true;
1.1803 +
1.1804 + // Disable incremental mode during a foreground collection.
1.1805 + ICMSDisabler icms_disabler;
1.1806 +
1.1807 + // release locks and wait for a notify from the background collector
1.1808 + // releasing the locks in only necessary for phases which
1.1809 + // do yields to improve the granularity of the collection.
1.1810 + assert_lock_strong(bitMapLock());
1.1811 + // We need to lock the Free list lock for the space that we are
1.1812 + // currently collecting.
1.1813 + assert(haveFreelistLocks(), "Must be holding free list locks");
1.1814 + bitMapLock()->unlock();
1.1815 + releaseFreelistLocks();
1.1816 + {
1.1817 + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1.1818 + if (_foregroundGCShouldWait) {
1.1819 + // We are going to be waiting for action for the CMS thread;
1.1820 + // it had better not be gone (for instance at shutdown)!
1.1821 + assert(ConcurrentMarkSweepThread::cmst() != NULL,
1.1822 + "CMS thread must be running");
1.1823 + // Wait here until the background collector gives us the go-ahead
1.1824 + ConcurrentMarkSweepThread::clear_CMS_flag(
1.1825 + ConcurrentMarkSweepThread::CMS_vm_has_token); // release token
1.1826 + // Get a possibly blocked CMS thread going:
1.1827 + // Note that we set _foregroundGCIsActive true above,
1.1828 + // without protection of the CGC_lock.
1.1829 + CGC_lock->notify();
1.1830 + assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(),
1.1831 + "Possible deadlock");
1.1832 + while (_foregroundGCShouldWait) {
1.1833 + // wait for notification
1.1834 + CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1.1835 + // Possibility of delay/starvation here, since CMS token does
1.1836 + // not know to give priority to VM thread? Actually, i think
1.1837 + // there wouldn't be any delay/starvation, but the proof of
1.1838 + // that "fact" (?) appears non-trivial. XXX 20011219YSR
1.1839 + }
1.1840 + ConcurrentMarkSweepThread::set_CMS_flag(
1.1841 + ConcurrentMarkSweepThread::CMS_vm_has_token);
1.1842 + }
1.1843 + }
1.1844 + // The CMS_token is already held. Get back the other locks.
1.1845 + assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1.1846 + "VM thread should have CMS token");
1.1847 + getFreelistLocks();
1.1848 + bitMapLock()->lock_without_safepoint_check();
1.1849 + if (TraceCMSState) {
1.1850 + gclog_or_tty->print_cr("CMS foreground collector has asked for control "
1.1851 + INTPTR_FORMAT " with first state %d", Thread::current(), first_state);
1.1852 + gclog_or_tty->print_cr(" gets control with state %d", _collectorState);
1.1853 + }
1.1854 +
1.1855 + // Check if we need to do a compaction, or if not, whether
1.1856 + // we need to start the mark-sweep from scratch.
1.1857 + bool should_compact = false;
1.1858 + bool should_start_over = false;
1.1859 + decide_foreground_collection_type(clear_all_soft_refs,
1.1860 + &should_compact, &should_start_over);
1.1861 +
1.1862 +NOT_PRODUCT(
1.1863 + if (RotateCMSCollectionTypes) {
1.1864 + if (_cmsGen->debug_collection_type() ==
1.1865 + ConcurrentMarkSweepGeneration::MSC_foreground_collection_type) {
1.1866 + should_compact = true;
1.1867 + } else if (_cmsGen->debug_collection_type() ==
1.1868 + ConcurrentMarkSweepGeneration::MS_foreground_collection_type) {
1.1869 + should_compact = false;
1.1870 + }
1.1871 + }
1.1872 +)
1.1873 +
1.1874 + if (first_state > Idling) {
1.1875 + report_concurrent_mode_interruption();
1.1876 + }
1.1877 +
1.1878 + set_did_compact(should_compact);
1.1879 + if (should_compact) {
1.1880 + // If the collection is being acquired from the background
1.1881 + // collector, there may be references on the discovered
1.1882 + // references lists that have NULL referents (being those
1.1883 + // that were concurrently cleared by a mutator) or
1.1884 + // that are no longer active (having been enqueued concurrently
1.1885 + // by the mutator).
1.1886 + // Scrub the list of those references because Mark-Sweep-Compact
1.1887 + // code assumes referents are not NULL and that all discovered
1.1888 + // Reference objects are active.
1.1889 + ref_processor()->clean_up_discovered_references();
1.1890 +
1.1891 + if (first_state > Idling) {
1.1892 + save_heap_summary();
1.1893 + }
1.1894 +
1.1895 + do_compaction_work(clear_all_soft_refs);
1.1896 +
1.1897 + // Has the GC time limit been exceeded?
1.1898 + DefNewGeneration* young_gen = _young_gen->as_DefNewGeneration();
1.1899 + size_t max_eden_size = young_gen->max_capacity() -
1.1900 + young_gen->to()->capacity() -
1.1901 + young_gen->from()->capacity();
1.1902 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.1903 + GCCause::Cause gc_cause = gch->gc_cause();
1.1904 + size_policy()->check_gc_overhead_limit(_young_gen->used(),
1.1905 + young_gen->eden()->used(),
1.1906 + _cmsGen->max_capacity(),
1.1907 + max_eden_size,
1.1908 + full,
1.1909 + gc_cause,
1.1910 + gch->collector_policy());
1.1911 + } else {
1.1912 + do_mark_sweep_work(clear_all_soft_refs, first_state,
1.1913 + should_start_over);
1.1914 + }
1.1915 + // Reset the expansion cause, now that we just completed
1.1916 + // a collection cycle.
1.1917 + clear_expansion_cause();
1.1918 + _foregroundGCIsActive = false;
1.1919 + return;
1.1920 +}
1.1921 +
1.1922 +// Resize the tenured generation
1.1923 +// after obtaining the free list locks for the
1.1924 +// two generations.
1.1925 +void CMSCollector::compute_new_size() {
1.1926 + assert_locked_or_safepoint(Heap_lock);
1.1927 + FreelistLocker z(this);
1.1928 + MetaspaceGC::compute_new_size();
1.1929 + _cmsGen->compute_new_size_free_list();
1.1930 +}
1.1931 +
1.1932 +// A work method used by foreground collection to determine
1.1933 +// what type of collection (compacting or not, continuing or fresh)
1.1934 +// it should do.
1.1935 +// NOTE: the intent is to make UseCMSCompactAtFullCollection
1.1936 +// and CMSCompactWhenClearAllSoftRefs the default in the future
1.1937 +// and do away with the flags after a suitable period.
1.1938 +void CMSCollector::decide_foreground_collection_type(
1.1939 + bool clear_all_soft_refs, bool* should_compact,
1.1940 + bool* should_start_over) {
1.1941 + // Normally, we'll compact only if the UseCMSCompactAtFullCollection
1.1942 + // flag is set, and we have either requested a System.gc() or
1.1943 + // the number of full gc's since the last concurrent cycle
1.1944 + // has exceeded the threshold set by CMSFullGCsBeforeCompaction,
1.1945 + // or if an incremental collection has failed
1.1946 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.1947 + assert(gch->collector_policy()->is_two_generation_policy(),
1.1948 + "You may want to check the correctness of the following");
1.1949 + // Inform cms gen if this was due to partial collection failing.
1.1950 + // The CMS gen may use this fact to determine its expansion policy.
1.1951 + if (gch->incremental_collection_will_fail(false /* don't consult_young */)) {
1.1952 + assert(!_cmsGen->incremental_collection_failed(),
1.1953 + "Should have been noticed, reacted to and cleared");
1.1954 + _cmsGen->set_incremental_collection_failed();
1.1955 + }
1.1956 + *should_compact =
1.1957 + UseCMSCompactAtFullCollection &&
1.1958 + ((_full_gcs_since_conc_gc >= CMSFullGCsBeforeCompaction) ||
1.1959 + GCCause::is_user_requested_gc(gch->gc_cause()) ||
1.1960 + gch->incremental_collection_will_fail(true /* consult_young */));
1.1961 + *should_start_over = false;
1.1962 + if (clear_all_soft_refs && !*should_compact) {
1.1963 + // We are about to do a last ditch collection attempt
1.1964 + // so it would normally make sense to do a compaction
1.1965 + // to reclaim as much space as possible.
1.1966 + if (CMSCompactWhenClearAllSoftRefs) {
1.1967 + // Default: The rationale is that in this case either
1.1968 + // we are past the final marking phase, in which case
1.1969 + // we'd have to start over, or so little has been done
1.1970 + // that there's little point in saving that work. Compaction
1.1971 + // appears to be the sensible choice in either case.
1.1972 + *should_compact = true;
1.1973 + } else {
1.1974 + // We have been asked to clear all soft refs, but not to
1.1975 + // compact. Make sure that we aren't past the final checkpoint
1.1976 + // phase, for that is where we process soft refs. If we are already
1.1977 + // past that phase, we'll need to redo the refs discovery phase and
1.1978 + // if necessary clear soft refs that weren't previously
1.1979 + // cleared. We do so by remembering the phase in which
1.1980 + // we came in, and if we are past the refs processing
1.1981 + // phase, we'll choose to just redo the mark-sweep
1.1982 + // collection from scratch.
1.1983 + if (_collectorState > FinalMarking) {
1.1984 + // We are past the refs processing phase;
1.1985 + // start over and do a fresh synchronous CMS cycle
1.1986 + _collectorState = Resetting; // skip to reset to start new cycle
1.1987 + reset(false /* == !asynch */);
1.1988 + *should_start_over = true;
1.1989 + } // else we can continue a possibly ongoing current cycle
1.1990 + }
1.1991 + }
1.1992 +}
1.1993 +
1.1994 +// A work method used by the foreground collector to do
1.1995 +// a mark-sweep-compact.
1.1996 +void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
1.1997 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.1998 +
1.1999 + STWGCTimer* gc_timer = GenMarkSweep::gc_timer();
1.2000 + gc_timer->register_gc_start();
1.2001 +
1.2002 + SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer();
1.2003 + gc_tracer->report_gc_start(gch->gc_cause(), gc_timer->gc_start());
1.2004 +
1.2005 + GCTraceTime t("CMS:MSC ", PrintGCDetails && Verbose, true, NULL);
1.2006 + if (PrintGC && Verbose && !(GCCause::is_user_requested_gc(gch->gc_cause()))) {
1.2007 + gclog_or_tty->print_cr("Compact ConcurrentMarkSweepGeneration after %d "
1.2008 + "collections passed to foreground collector", _full_gcs_since_conc_gc);
1.2009 + }
1.2010 +
1.2011 + // Sample collection interval time and reset for collection pause.
1.2012 + if (UseAdaptiveSizePolicy) {
1.2013 + size_policy()->msc_collection_begin();
1.2014 + }
1.2015 +
1.2016 + // Temporarily widen the span of the weak reference processing to
1.2017 + // the entire heap.
1.2018 + MemRegion new_span(GenCollectedHeap::heap()->reserved_region());
1.2019 + ReferenceProcessorSpanMutator rp_mut_span(ref_processor(), new_span);
1.2020 + // Temporarily, clear the "is_alive_non_header" field of the
1.2021 + // reference processor.
1.2022 + ReferenceProcessorIsAliveMutator rp_mut_closure(ref_processor(), NULL);
1.2023 + // Temporarily make reference _processing_ single threaded (non-MT).
1.2024 + ReferenceProcessorMTProcMutator rp_mut_mt_processing(ref_processor(), false);
1.2025 + // Temporarily make refs discovery atomic
1.2026 + ReferenceProcessorAtomicMutator rp_mut_atomic(ref_processor(), true);
1.2027 + // Temporarily make reference _discovery_ single threaded (non-MT)
1.2028 + ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false);
1.2029 +
1.2030 + ref_processor()->set_enqueuing_is_done(false);
1.2031 + ref_processor()->enable_discovery(false /*verify_disabled*/, false /*check_no_refs*/);
1.2032 + ref_processor()->setup_policy(clear_all_soft_refs);
1.2033 + // If an asynchronous collection finishes, the _modUnionTable is
1.2034 + // all clear. If we are assuming the collection from an asynchronous
1.2035 + // collection, clear the _modUnionTable.
1.2036 + assert(_collectorState != Idling || _modUnionTable.isAllClear(),
1.2037 + "_modUnionTable should be clear if the baton was not passed");
1.2038 + _modUnionTable.clear_all();
1.2039 + assert(_collectorState != Idling || _ct->klass_rem_set()->mod_union_is_clear(),
1.2040 + "mod union for klasses should be clear if the baton was passed");
1.2041 + _ct->klass_rem_set()->clear_mod_union();
1.2042 +
1.2043 + // We must adjust the allocation statistics being maintained
1.2044 + // in the free list space. We do so by reading and clearing
1.2045 + // the sweep timer and updating the block flux rate estimates below.
1.2046 + assert(!_intra_sweep_timer.is_active(), "_intra_sweep_timer should be inactive");
1.2047 + if (_inter_sweep_timer.is_active()) {
1.2048 + _inter_sweep_timer.stop();
1.2049 + // Note that we do not use this sample to update the _inter_sweep_estimate.
1.2050 + _cmsGen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()),
1.2051 + _inter_sweep_estimate.padded_average(),
1.2052 + _intra_sweep_estimate.padded_average());
1.2053 + }
1.2054 +
1.2055 + GenMarkSweep::invoke_at_safepoint(_cmsGen->level(),
1.2056 + ref_processor(), clear_all_soft_refs);
1.2057 + #ifdef ASSERT
1.2058 + CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
1.2059 + size_t free_size = cms_space->free();
1.2060 + assert(free_size ==
1.2061 + pointer_delta(cms_space->end(), cms_space->compaction_top())
1.2062 + * HeapWordSize,
1.2063 + "All the free space should be compacted into one chunk at top");
1.2064 + assert(cms_space->dictionary()->total_chunk_size(
1.2065 + debug_only(cms_space->freelistLock())) == 0 ||
1.2066 + cms_space->totalSizeInIndexedFreeLists() == 0,
1.2067 + "All the free space should be in a single chunk");
1.2068 + size_t num = cms_space->totalCount();
1.2069 + assert((free_size == 0 && num == 0) ||
1.2070 + (free_size > 0 && (num == 1 || num == 2)),
1.2071 + "There should be at most 2 free chunks after compaction");
1.2072 + #endif // ASSERT
1.2073 + _collectorState = Resetting;
1.2074 + assert(_restart_addr == NULL,
1.2075 + "Should have been NULL'd before baton was passed");
1.2076 + reset(false /* == !asynch */);
1.2077 + _cmsGen->reset_after_compaction();
1.2078 + _concurrent_cycles_since_last_unload = 0;
1.2079 +
1.2080 + // Clear any data recorded in the PLAB chunk arrays.
1.2081 + if (_survivor_plab_array != NULL) {
1.2082 + reset_survivor_plab_arrays();
1.2083 + }
1.2084 +
1.2085 + // Adjust the per-size allocation stats for the next epoch.
1.2086 + _cmsGen->cmsSpace()->endSweepFLCensus(sweep_count() /* fake */);
1.2087 + // Restart the "inter sweep timer" for the next epoch.
1.2088 + _inter_sweep_timer.reset();
1.2089 + _inter_sweep_timer.start();
1.2090 +
1.2091 + // Sample collection pause time and reset for collection interval.
1.2092 + if (UseAdaptiveSizePolicy) {
1.2093 + size_policy()->msc_collection_end(gch->gc_cause());
1.2094 + }
1.2095 +
1.2096 + gc_timer->register_gc_end();
1.2097 +
1.2098 + gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions());
1.2099 +
1.2100 + // For a mark-sweep-compact, compute_new_size() will be called
1.2101 + // in the heap's do_collection() method.
1.2102 +}
1.2103 +
1.2104 +// A work method used by the foreground collector to do
1.2105 +// a mark-sweep, after taking over from a possibly on-going
1.2106 +// concurrent mark-sweep collection.
1.2107 +void CMSCollector::do_mark_sweep_work(bool clear_all_soft_refs,
1.2108 + CollectorState first_state, bool should_start_over) {
1.2109 + if (PrintGC && Verbose) {
1.2110 + gclog_or_tty->print_cr("Pass concurrent collection to foreground "
1.2111 + "collector with count %d",
1.2112 + _full_gcs_since_conc_gc);
1.2113 + }
1.2114 + switch (_collectorState) {
1.2115 + case Idling:
1.2116 + if (first_state == Idling || should_start_over) {
1.2117 + // The background GC was not active, or should
1.2118 + // restarted from scratch; start the cycle.
1.2119 + _collectorState = InitialMarking;
1.2120 + }
1.2121 + // If first_state was not Idling, then a background GC
1.2122 + // was in progress and has now finished. No need to do it
1.2123 + // again. Leave the state as Idling.
1.2124 + break;
1.2125 + case Precleaning:
1.2126 + // In the foreground case don't do the precleaning since
1.2127 + // it is not done concurrently and there is extra work
1.2128 + // required.
1.2129 + _collectorState = FinalMarking;
1.2130 + }
1.2131 + collect_in_foreground(clear_all_soft_refs, GenCollectedHeap::heap()->gc_cause());
1.2132 +
1.2133 + // For a mark-sweep, compute_new_size() will be called
1.2134 + // in the heap's do_collection() method.
1.2135 +}
1.2136 +
1.2137 +
1.2138 +void CMSCollector::print_eden_and_survivor_chunk_arrays() {
1.2139 + DefNewGeneration* dng = _young_gen->as_DefNewGeneration();
1.2140 + EdenSpace* eden_space = dng->eden();
1.2141 + ContiguousSpace* from_space = dng->from();
1.2142 + ContiguousSpace* to_space = dng->to();
1.2143 + // Eden
1.2144 + if (_eden_chunk_array != NULL) {
1.2145 + gclog_or_tty->print_cr("eden " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1.2146 + eden_space->bottom(), eden_space->top(),
1.2147 + eden_space->end(), eden_space->capacity());
1.2148 + gclog_or_tty->print_cr("_eden_chunk_index=" SIZE_FORMAT ", "
1.2149 + "_eden_chunk_capacity=" SIZE_FORMAT,
1.2150 + _eden_chunk_index, _eden_chunk_capacity);
1.2151 + for (size_t i = 0; i < _eden_chunk_index; i++) {
1.2152 + gclog_or_tty->print_cr("_eden_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT,
1.2153 + i, _eden_chunk_array[i]);
1.2154 + }
1.2155 + }
1.2156 + // Survivor
1.2157 + if (_survivor_chunk_array != NULL) {
1.2158 + gclog_or_tty->print_cr("survivor " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1.2159 + from_space->bottom(), from_space->top(),
1.2160 + from_space->end(), from_space->capacity());
1.2161 + gclog_or_tty->print_cr("_survivor_chunk_index=" SIZE_FORMAT ", "
1.2162 + "_survivor_chunk_capacity=" SIZE_FORMAT,
1.2163 + _survivor_chunk_index, _survivor_chunk_capacity);
1.2164 + for (size_t i = 0; i < _survivor_chunk_index; i++) {
1.2165 + gclog_or_tty->print_cr("_survivor_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT,
1.2166 + i, _survivor_chunk_array[i]);
1.2167 + }
1.2168 + }
1.2169 +}
1.2170 +
1.2171 +void CMSCollector::getFreelistLocks() const {
1.2172 + // Get locks for all free lists in all generations that this
1.2173 + // collector is responsible for
1.2174 + _cmsGen->freelistLock()->lock_without_safepoint_check();
1.2175 +}
1.2176 +
1.2177 +void CMSCollector::releaseFreelistLocks() const {
1.2178 + // Release locks for all free lists in all generations that this
1.2179 + // collector is responsible for
1.2180 + _cmsGen->freelistLock()->unlock();
1.2181 +}
1.2182 +
1.2183 +bool CMSCollector::haveFreelistLocks() const {
1.2184 + // Check locks for all free lists in all generations that this
1.2185 + // collector is responsible for
1.2186 + assert_lock_strong(_cmsGen->freelistLock());
1.2187 + PRODUCT_ONLY(ShouldNotReachHere());
1.2188 + return true;
1.2189 +}
1.2190 +
1.2191 +// A utility class that is used by the CMS collector to
1.2192 +// temporarily "release" the foreground collector from its
1.2193 +// usual obligation to wait for the background collector to
1.2194 +// complete an ongoing phase before proceeding.
1.2195 +class ReleaseForegroundGC: public StackObj {
1.2196 + private:
1.2197 + CMSCollector* _c;
1.2198 + public:
1.2199 + ReleaseForegroundGC(CMSCollector* c) : _c(c) {
1.2200 + assert(_c->_foregroundGCShouldWait, "Else should not need to call");
1.2201 + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1.2202 + // allow a potentially blocked foreground collector to proceed
1.2203 + _c->_foregroundGCShouldWait = false;
1.2204 + if (_c->_foregroundGCIsActive) {
1.2205 + CGC_lock->notify();
1.2206 + }
1.2207 + assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.2208 + "Possible deadlock");
1.2209 + }
1.2210 +
1.2211 + ~ReleaseForegroundGC() {
1.2212 + assert(!_c->_foregroundGCShouldWait, "Usage protocol violation?");
1.2213 + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1.2214 + _c->_foregroundGCShouldWait = true;
1.2215 + }
1.2216 +};
1.2217 +
1.2218 +// There are separate collect_in_background and collect_in_foreground because of
1.2219 +// the different locking requirements of the background collector and the
1.2220 +// foreground collector. There was originally an attempt to share
1.2221 +// one "collect" method between the background collector and the foreground
1.2222 +// collector but the if-then-else required made it cleaner to have
1.2223 +// separate methods.
1.2224 +void CMSCollector::collect_in_background(bool clear_all_soft_refs, GCCause::Cause cause) {
1.2225 + assert(Thread::current()->is_ConcurrentGC_thread(),
1.2226 + "A CMS asynchronous collection is only allowed on a CMS thread.");
1.2227 +
1.2228 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.2229 + {
1.2230 + bool safepoint_check = Mutex::_no_safepoint_check_flag;
1.2231 + MutexLockerEx hl(Heap_lock, safepoint_check);
1.2232 + FreelistLocker fll(this);
1.2233 + MutexLockerEx x(CGC_lock, safepoint_check);
1.2234 + if (_foregroundGCIsActive || !UseAsyncConcMarkSweepGC) {
1.2235 + // The foreground collector is active or we're
1.2236 + // not using asynchronous collections. Skip this
1.2237 + // background collection.
1.2238 + assert(!_foregroundGCShouldWait, "Should be clear");
1.2239 + return;
1.2240 + } else {
1.2241 + assert(_collectorState == Idling, "Should be idling before start.");
1.2242 + _collectorState = InitialMarking;
1.2243 + register_gc_start(cause);
1.2244 + // Reset the expansion cause, now that we are about to begin
1.2245 + // a new cycle.
1.2246 + clear_expansion_cause();
1.2247 +
1.2248 + // Clear the MetaspaceGC flag since a concurrent collection
1.2249 + // is starting but also clear it after the collection.
1.2250 + MetaspaceGC::set_should_concurrent_collect(false);
1.2251 + }
1.2252 + // Decide if we want to enable class unloading as part of the
1.2253 + // ensuing concurrent GC cycle.
1.2254 + update_should_unload_classes();
1.2255 + _full_gc_requested = false; // acks all outstanding full gc requests
1.2256 + _full_gc_cause = GCCause::_no_gc;
1.2257 + // Signal that we are about to start a collection
1.2258 + gch->increment_total_full_collections(); // ... starting a collection cycle
1.2259 + _collection_count_start = gch->total_full_collections();
1.2260 + }
1.2261 +
1.2262 + // Used for PrintGC
1.2263 + size_t prev_used;
1.2264 + if (PrintGC && Verbose) {
1.2265 + prev_used = _cmsGen->used(); // XXXPERM
1.2266 + }
1.2267 +
1.2268 + // The change of the collection state is normally done at this level;
1.2269 + // the exceptions are phases that are executed while the world is
1.2270 + // stopped. For those phases the change of state is done while the
1.2271 + // world is stopped. For baton passing purposes this allows the
1.2272 + // background collector to finish the phase and change state atomically.
1.2273 + // The foreground collector cannot wait on a phase that is done
1.2274 + // while the world is stopped because the foreground collector already
1.2275 + // has the world stopped and would deadlock.
1.2276 + while (_collectorState != Idling) {
1.2277 + if (TraceCMSState) {
1.2278 + gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
1.2279 + Thread::current(), _collectorState);
1.2280 + }
1.2281 + // The foreground collector
1.2282 + // holds the Heap_lock throughout its collection.
1.2283 + // holds the CMS token (but not the lock)
1.2284 + // except while it is waiting for the background collector to yield.
1.2285 + //
1.2286 + // The foreground collector should be blocked (not for long)
1.2287 + // if the background collector is about to start a phase
1.2288 + // executed with world stopped. If the background
1.2289 + // collector has already started such a phase, the
1.2290 + // foreground collector is blocked waiting for the
1.2291 + // Heap_lock. The stop-world phases (InitialMarking and FinalMarking)
1.2292 + // are executed in the VM thread.
1.2293 + //
1.2294 + // The locking order is
1.2295 + // PendingListLock (PLL) -- if applicable (FinalMarking)
1.2296 + // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue())
1.2297 + // CMS token (claimed in
1.2298 + // stop_world_and_do() -->
1.2299 + // safepoint_synchronize() -->
1.2300 + // CMSThread::synchronize())
1.2301 +
1.2302 + {
1.2303 + // Check if the FG collector wants us to yield.
1.2304 + CMSTokenSync x(true); // is cms thread
1.2305 + if (waitForForegroundGC()) {
1.2306 + // We yielded to a foreground GC, nothing more to be
1.2307 + // done this round.
1.2308 + assert(_foregroundGCShouldWait == false, "We set it to false in "
1.2309 + "waitForForegroundGC()");
1.2310 + if (TraceCMSState) {
1.2311 + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1.2312 + " exiting collection CMS state %d",
1.2313 + Thread::current(), _collectorState);
1.2314 + }
1.2315 + return;
1.2316 + } else {
1.2317 + // The background collector can run but check to see if the
1.2318 + // foreground collector has done a collection while the
1.2319 + // background collector was waiting to get the CGC_lock
1.2320 + // above. If yes, break so that _foregroundGCShouldWait
1.2321 + // is cleared before returning.
1.2322 + if (_collectorState == Idling) {
1.2323 + break;
1.2324 + }
1.2325 + }
1.2326 + }
1.2327 +
1.2328 + assert(_foregroundGCShouldWait, "Foreground collector, if active, "
1.2329 + "should be waiting");
1.2330 +
1.2331 + switch (_collectorState) {
1.2332 + case InitialMarking:
1.2333 + {
1.2334 + ReleaseForegroundGC x(this);
1.2335 + stats().record_cms_begin();
1.2336 + VM_CMS_Initial_Mark initial_mark_op(this);
1.2337 + VMThread::execute(&initial_mark_op);
1.2338 + }
1.2339 + // The collector state may be any legal state at this point
1.2340 + // since the background collector may have yielded to the
1.2341 + // foreground collector.
1.2342 + break;
1.2343 + case Marking:
1.2344 + // initial marking in checkpointRootsInitialWork has been completed
1.2345 + if (markFromRoots(true)) { // we were successful
1.2346 + assert(_collectorState == Precleaning, "Collector state should "
1.2347 + "have changed");
1.2348 + } else {
1.2349 + assert(_foregroundGCIsActive, "Internal state inconsistency");
1.2350 + }
1.2351 + break;
1.2352 + case Precleaning:
1.2353 + if (UseAdaptiveSizePolicy) {
1.2354 + size_policy()->concurrent_precleaning_begin();
1.2355 + }
1.2356 + // marking from roots in markFromRoots has been completed
1.2357 + preclean();
1.2358 + if (UseAdaptiveSizePolicy) {
1.2359 + size_policy()->concurrent_precleaning_end();
1.2360 + }
1.2361 + assert(_collectorState == AbortablePreclean ||
1.2362 + _collectorState == FinalMarking,
1.2363 + "Collector state should have changed");
1.2364 + break;
1.2365 + case AbortablePreclean:
1.2366 + if (UseAdaptiveSizePolicy) {
1.2367 + size_policy()->concurrent_phases_resume();
1.2368 + }
1.2369 + abortable_preclean();
1.2370 + if (UseAdaptiveSizePolicy) {
1.2371 + size_policy()->concurrent_precleaning_end();
1.2372 + }
1.2373 + assert(_collectorState == FinalMarking, "Collector state should "
1.2374 + "have changed");
1.2375 + break;
1.2376 + case FinalMarking:
1.2377 + {
1.2378 + ReleaseForegroundGC x(this);
1.2379 +
1.2380 + VM_CMS_Final_Remark final_remark_op(this);
1.2381 + VMThread::execute(&final_remark_op);
1.2382 + }
1.2383 + assert(_foregroundGCShouldWait, "block post-condition");
1.2384 + break;
1.2385 + case Sweeping:
1.2386 + if (UseAdaptiveSizePolicy) {
1.2387 + size_policy()->concurrent_sweeping_begin();
1.2388 + }
1.2389 + // final marking in checkpointRootsFinal has been completed
1.2390 + sweep(true);
1.2391 + assert(_collectorState == Resizing, "Collector state change "
1.2392 + "to Resizing must be done under the free_list_lock");
1.2393 + _full_gcs_since_conc_gc = 0;
1.2394 +
1.2395 + // Stop the timers for adaptive size policy for the concurrent phases
1.2396 + if (UseAdaptiveSizePolicy) {
1.2397 + size_policy()->concurrent_sweeping_end();
1.2398 + size_policy()->concurrent_phases_end(gch->gc_cause(),
1.2399 + gch->prev_gen(_cmsGen)->capacity(),
1.2400 + _cmsGen->free());
1.2401 + }
1.2402 +
1.2403 + case Resizing: {
1.2404 + // Sweeping has been completed...
1.2405 + // At this point the background collection has completed.
1.2406 + // Don't move the call to compute_new_size() down
1.2407 + // into code that might be executed if the background
1.2408 + // collection was preempted.
1.2409 + {
1.2410 + ReleaseForegroundGC x(this); // unblock FG collection
1.2411 + MutexLockerEx y(Heap_lock, Mutex::_no_safepoint_check_flag);
1.2412 + CMSTokenSync z(true); // not strictly needed.
1.2413 + if (_collectorState == Resizing) {
1.2414 + compute_new_size();
1.2415 + save_heap_summary();
1.2416 + _collectorState = Resetting;
1.2417 + } else {
1.2418 + assert(_collectorState == Idling, "The state should only change"
1.2419 + " because the foreground collector has finished the collection");
1.2420 + }
1.2421 + }
1.2422 + break;
1.2423 + }
1.2424 + case Resetting:
1.2425 + // CMS heap resizing has been completed
1.2426 + reset(true);
1.2427 + assert(_collectorState == Idling, "Collector state should "
1.2428 + "have changed");
1.2429 +
1.2430 + MetaspaceGC::set_should_concurrent_collect(false);
1.2431 +
1.2432 + stats().record_cms_end();
1.2433 + // Don't move the concurrent_phases_end() and compute_new_size()
1.2434 + // calls to here because a preempted background collection
1.2435 + // has it's state set to "Resetting".
1.2436 + break;
1.2437 + case Idling:
1.2438 + default:
1.2439 + ShouldNotReachHere();
1.2440 + break;
1.2441 + }
1.2442 + if (TraceCMSState) {
1.2443 + gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
1.2444 + Thread::current(), _collectorState);
1.2445 + }
1.2446 + assert(_foregroundGCShouldWait, "block post-condition");
1.2447 + }
1.2448 +
1.2449 + // Should this be in gc_epilogue?
1.2450 + collector_policy()->counters()->update_counters();
1.2451 +
1.2452 + {
1.2453 + // Clear _foregroundGCShouldWait and, in the event that the
1.2454 + // foreground collector is waiting, notify it, before
1.2455 + // returning.
1.2456 + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1.2457 + _foregroundGCShouldWait = false;
1.2458 + if (_foregroundGCIsActive) {
1.2459 + CGC_lock->notify();
1.2460 + }
1.2461 + assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.2462 + "Possible deadlock");
1.2463 + }
1.2464 + if (TraceCMSState) {
1.2465 + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1.2466 + " exiting collection CMS state %d",
1.2467 + Thread::current(), _collectorState);
1.2468 + }
1.2469 + if (PrintGC && Verbose) {
1.2470 + _cmsGen->print_heap_change(prev_used);
1.2471 + }
1.2472 +}
1.2473 +
1.2474 +void CMSCollector::register_foreground_gc_start(GCCause::Cause cause) {
1.2475 + if (!_cms_start_registered) {
1.2476 + register_gc_start(cause);
1.2477 + }
1.2478 +}
1.2479 +
1.2480 +void CMSCollector::register_gc_start(GCCause::Cause cause) {
1.2481 + _cms_start_registered = true;
1.2482 + _gc_timer_cm->register_gc_start();
1.2483 + _gc_tracer_cm->report_gc_start(cause, _gc_timer_cm->gc_start());
1.2484 +}
1.2485 +
1.2486 +void CMSCollector::register_gc_end() {
1.2487 + if (_cms_start_registered) {
1.2488 + report_heap_summary(GCWhen::AfterGC);
1.2489 +
1.2490 + _gc_timer_cm->register_gc_end();
1.2491 + _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
1.2492 + _cms_start_registered = false;
1.2493 + }
1.2494 +}
1.2495 +
1.2496 +void CMSCollector::save_heap_summary() {
1.2497 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.2498 + _last_heap_summary = gch->create_heap_summary();
1.2499 + _last_metaspace_summary = gch->create_metaspace_summary();
1.2500 +}
1.2501 +
1.2502 +void CMSCollector::report_heap_summary(GCWhen::Type when) {
1.2503 + _gc_tracer_cm->report_gc_heap_summary(when, _last_heap_summary);
1.2504 + _gc_tracer_cm->report_metaspace_summary(when, _last_metaspace_summary);
1.2505 +}
1.2506 +
1.2507 +void CMSCollector::collect_in_foreground(bool clear_all_soft_refs, GCCause::Cause cause) {
1.2508 + assert(_foregroundGCIsActive && !_foregroundGCShouldWait,
1.2509 + "Foreground collector should be waiting, not executing");
1.2510 + assert(Thread::current()->is_VM_thread(), "A foreground collection"
1.2511 + "may only be done by the VM Thread with the world stopped");
1.2512 + assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1.2513 + "VM thread should have CMS token");
1.2514 +
1.2515 + NOT_PRODUCT(GCTraceTime t("CMS:MS (foreground) ", PrintGCDetails && Verbose,
1.2516 + true, NULL);)
1.2517 + if (UseAdaptiveSizePolicy) {
1.2518 + size_policy()->ms_collection_begin();
1.2519 + }
1.2520 + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact);
1.2521 +
1.2522 + HandleMark hm; // Discard invalid handles created during verification
1.2523 +
1.2524 + if (VerifyBeforeGC &&
1.2525 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2526 + Universe::verify();
1.2527 + }
1.2528 +
1.2529 + // Snapshot the soft reference policy to be used in this collection cycle.
1.2530 + ref_processor()->setup_policy(clear_all_soft_refs);
1.2531 +
1.2532 + // Decide if class unloading should be done
1.2533 + update_should_unload_classes();
1.2534 +
1.2535 + bool init_mark_was_synchronous = false; // until proven otherwise
1.2536 + while (_collectorState != Idling) {
1.2537 + if (TraceCMSState) {
1.2538 + gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
1.2539 + Thread::current(), _collectorState);
1.2540 + }
1.2541 + switch (_collectorState) {
1.2542 + case InitialMarking:
1.2543 + register_foreground_gc_start(cause);
1.2544 + init_mark_was_synchronous = true; // fact to be exploited in re-mark
1.2545 + checkpointRootsInitial(false);
1.2546 + assert(_collectorState == Marking, "Collector state should have changed"
1.2547 + " within checkpointRootsInitial()");
1.2548 + break;
1.2549 + case Marking:
1.2550 + // initial marking in checkpointRootsInitialWork has been completed
1.2551 + if (VerifyDuringGC &&
1.2552 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2553 + Universe::verify("Verify before initial mark: ");
1.2554 + }
1.2555 + {
1.2556 + bool res = markFromRoots(false);
1.2557 + assert(res && _collectorState == FinalMarking, "Collector state should "
1.2558 + "have changed");
1.2559 + break;
1.2560 + }
1.2561 + case FinalMarking:
1.2562 + if (VerifyDuringGC &&
1.2563 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2564 + Universe::verify("Verify before re-mark: ");
1.2565 + }
1.2566 + checkpointRootsFinal(false, clear_all_soft_refs,
1.2567 + init_mark_was_synchronous);
1.2568 + assert(_collectorState == Sweeping, "Collector state should not "
1.2569 + "have changed within checkpointRootsFinal()");
1.2570 + break;
1.2571 + case Sweeping:
1.2572 + // final marking in checkpointRootsFinal has been completed
1.2573 + if (VerifyDuringGC &&
1.2574 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2575 + Universe::verify("Verify before sweep: ");
1.2576 + }
1.2577 + sweep(false);
1.2578 + assert(_collectorState == Resizing, "Incorrect state");
1.2579 + break;
1.2580 + case Resizing: {
1.2581 + // Sweeping has been completed; the actual resize in this case
1.2582 + // is done separately; nothing to be done in this state.
1.2583 + _collectorState = Resetting;
1.2584 + break;
1.2585 + }
1.2586 + case Resetting:
1.2587 + // The heap has been resized.
1.2588 + if (VerifyDuringGC &&
1.2589 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2590 + Universe::verify("Verify before reset: ");
1.2591 + }
1.2592 + save_heap_summary();
1.2593 + reset(false);
1.2594 + assert(_collectorState == Idling, "Collector state should "
1.2595 + "have changed");
1.2596 + break;
1.2597 + case Precleaning:
1.2598 + case AbortablePreclean:
1.2599 + // Elide the preclean phase
1.2600 + _collectorState = FinalMarking;
1.2601 + break;
1.2602 + default:
1.2603 + ShouldNotReachHere();
1.2604 + }
1.2605 + if (TraceCMSState) {
1.2606 + gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
1.2607 + Thread::current(), _collectorState);
1.2608 + }
1.2609 + }
1.2610 +
1.2611 + if (UseAdaptiveSizePolicy) {
1.2612 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.2613 + size_policy()->ms_collection_end(gch->gc_cause());
1.2614 + }
1.2615 +
1.2616 + if (VerifyAfterGC &&
1.2617 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.2618 + Universe::verify();
1.2619 + }
1.2620 + if (TraceCMSState) {
1.2621 + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1.2622 + " exiting collection CMS state %d",
1.2623 + Thread::current(), _collectorState);
1.2624 + }
1.2625 +}
1.2626 +
1.2627 +bool CMSCollector::waitForForegroundGC() {
1.2628 + bool res = false;
1.2629 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.2630 + "CMS thread should have CMS token");
1.2631 + // Block the foreground collector until the
1.2632 + // background collectors decides whether to
1.2633 + // yield.
1.2634 + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1.2635 + _foregroundGCShouldWait = true;
1.2636 + if (_foregroundGCIsActive) {
1.2637 + // The background collector yields to the
1.2638 + // foreground collector and returns a value
1.2639 + // indicating that it has yielded. The foreground
1.2640 + // collector can proceed.
1.2641 + res = true;
1.2642 + _foregroundGCShouldWait = false;
1.2643 + ConcurrentMarkSweepThread::clear_CMS_flag(
1.2644 + ConcurrentMarkSweepThread::CMS_cms_has_token);
1.2645 + ConcurrentMarkSweepThread::set_CMS_flag(
1.2646 + ConcurrentMarkSweepThread::CMS_cms_wants_token);
1.2647 + // Get a possibly blocked foreground thread going
1.2648 + CGC_lock->notify();
1.2649 + if (TraceCMSState) {
1.2650 + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d",
1.2651 + Thread::current(), _collectorState);
1.2652 + }
1.2653 + while (_foregroundGCIsActive) {
1.2654 + CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1.2655 + }
1.2656 + ConcurrentMarkSweepThread::set_CMS_flag(
1.2657 + ConcurrentMarkSweepThread::CMS_cms_has_token);
1.2658 + ConcurrentMarkSweepThread::clear_CMS_flag(
1.2659 + ConcurrentMarkSweepThread::CMS_cms_wants_token);
1.2660 + }
1.2661 + if (TraceCMSState) {
1.2662 + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d",
1.2663 + Thread::current(), _collectorState);
1.2664 + }
1.2665 + return res;
1.2666 +}
1.2667 +
1.2668 +// Because of the need to lock the free lists and other structures in
1.2669 +// the collector, common to all the generations that the collector is
1.2670 +// collecting, we need the gc_prologues of individual CMS generations
1.2671 +// delegate to their collector. It may have been simpler had the
1.2672 +// current infrastructure allowed one to call a prologue on a
1.2673 +// collector. In the absence of that we have the generation's
1.2674 +// prologue delegate to the collector, which delegates back
1.2675 +// some "local" work to a worker method in the individual generations
1.2676 +// that it's responsible for collecting, while itself doing any
1.2677 +// work common to all generations it's responsible for. A similar
1.2678 +// comment applies to the gc_epilogue()'s.
1.2679 +// The role of the varaible _between_prologue_and_epilogue is to
1.2680 +// enforce the invocation protocol.
1.2681 +void CMSCollector::gc_prologue(bool full) {
1.2682 + // Call gc_prologue_work() for the CMSGen
1.2683 + // we are responsible for.
1.2684 +
1.2685 + // The following locking discipline assumes that we are only called
1.2686 + // when the world is stopped.
1.2687 + assert(SafepointSynchronize::is_at_safepoint(), "world is stopped assumption");
1.2688 +
1.2689 + // The CMSCollector prologue must call the gc_prologues for the
1.2690 + // "generations" that it's responsible
1.2691 + // for.
1.2692 +
1.2693 + assert( Thread::current()->is_VM_thread()
1.2694 + || ( CMSScavengeBeforeRemark
1.2695 + && Thread::current()->is_ConcurrentGC_thread()),
1.2696 + "Incorrect thread type for prologue execution");
1.2697 +
1.2698 + if (_between_prologue_and_epilogue) {
1.2699 + // We have already been invoked; this is a gc_prologue delegation
1.2700 + // from yet another CMS generation that we are responsible for, just
1.2701 + // ignore it since all relevant work has already been done.
1.2702 + return;
1.2703 + }
1.2704 +
1.2705 + // set a bit saying prologue has been called; cleared in epilogue
1.2706 + _between_prologue_and_epilogue = true;
1.2707 + // Claim locks for common data structures, then call gc_prologue_work()
1.2708 + // for each CMSGen.
1.2709 +
1.2710 + getFreelistLocks(); // gets free list locks on constituent spaces
1.2711 + bitMapLock()->lock_without_safepoint_check();
1.2712 +
1.2713 + // Should call gc_prologue_work() for all cms gens we are responsible for
1.2714 + bool duringMarking = _collectorState >= Marking
1.2715 + && _collectorState < Sweeping;
1.2716 +
1.2717 + // The young collections clear the modified oops state, which tells if
1.2718 + // there are any modified oops in the class. The remark phase also needs
1.2719 + // that information. Tell the young collection to save the union of all
1.2720 + // modified klasses.
1.2721 + if (duringMarking) {
1.2722 + _ct->klass_rem_set()->set_accumulate_modified_oops(true);
1.2723 + }
1.2724 +
1.2725 + bool registerClosure = duringMarking;
1.2726 +
1.2727 + ModUnionClosure* muc = CollectedHeap::use_parallel_gc_threads() ?
1.2728 + &_modUnionClosurePar
1.2729 + : &_modUnionClosure;
1.2730 + _cmsGen->gc_prologue_work(full, registerClosure, muc);
1.2731 +
1.2732 + if (!full) {
1.2733 + stats().record_gc0_begin();
1.2734 + }
1.2735 +}
1.2736 +
1.2737 +void ConcurrentMarkSweepGeneration::gc_prologue(bool full) {
1.2738 +
1.2739 + _capacity_at_prologue = capacity();
1.2740 + _used_at_prologue = used();
1.2741 +
1.2742 + // Delegate to CMScollector which knows how to coordinate between
1.2743 + // this and any other CMS generations that it is responsible for
1.2744 + // collecting.
1.2745 + collector()->gc_prologue(full);
1.2746 +}
1.2747 +
1.2748 +// This is a "private" interface for use by this generation's CMSCollector.
1.2749 +// Not to be called directly by any other entity (for instance,
1.2750 +// GenCollectedHeap, which calls the "public" gc_prologue method above).
1.2751 +void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
1.2752 + bool registerClosure, ModUnionClosure* modUnionClosure) {
1.2753 + assert(!incremental_collection_failed(), "Shouldn't be set yet");
1.2754 + assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,
1.2755 + "Should be NULL");
1.2756 + if (registerClosure) {
1.2757 + cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure);
1.2758 + }
1.2759 + cmsSpace()->gc_prologue();
1.2760 + // Clear stat counters
1.2761 + NOT_PRODUCT(
1.2762 + assert(_numObjectsPromoted == 0, "check");
1.2763 + assert(_numWordsPromoted == 0, "check");
1.2764 + if (Verbose && PrintGC) {
1.2765 + gclog_or_tty->print("Allocated "SIZE_FORMAT" objects, "
1.2766 + SIZE_FORMAT" bytes concurrently",
1.2767 + _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord));
1.2768 + }
1.2769 + _numObjectsAllocated = 0;
1.2770 + _numWordsAllocated = 0;
1.2771 + )
1.2772 +}
1.2773 +
1.2774 +void CMSCollector::gc_epilogue(bool full) {
1.2775 + // The following locking discipline assumes that we are only called
1.2776 + // when the world is stopped.
1.2777 + assert(SafepointSynchronize::is_at_safepoint(),
1.2778 + "world is stopped assumption");
1.2779 +
1.2780 + // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks
1.2781 + // if linear allocation blocks need to be appropriately marked to allow the
1.2782 + // the blocks to be parsable. We also check here whether we need to nudge the
1.2783 + // CMS collector thread to start a new cycle (if it's not already active).
1.2784 + assert( Thread::current()->is_VM_thread()
1.2785 + || ( CMSScavengeBeforeRemark
1.2786 + && Thread::current()->is_ConcurrentGC_thread()),
1.2787 + "Incorrect thread type for epilogue execution");
1.2788 +
1.2789 + if (!_between_prologue_and_epilogue) {
1.2790 + // We have already been invoked; this is a gc_epilogue delegation
1.2791 + // from yet another CMS generation that we are responsible for, just
1.2792 + // ignore it since all relevant work has already been done.
1.2793 + return;
1.2794 + }
1.2795 + assert(haveFreelistLocks(), "must have freelist locks");
1.2796 + assert_lock_strong(bitMapLock());
1.2797 +
1.2798 + _ct->klass_rem_set()->set_accumulate_modified_oops(false);
1.2799 +
1.2800 + _cmsGen->gc_epilogue_work(full);
1.2801 +
1.2802 + if (_collectorState == AbortablePreclean || _collectorState == Precleaning) {
1.2803 + // in case sampling was not already enabled, enable it
1.2804 + _start_sampling = true;
1.2805 + }
1.2806 + // reset _eden_chunk_array so sampling starts afresh
1.2807 + _eden_chunk_index = 0;
1.2808 +
1.2809 + size_t cms_used = _cmsGen->cmsSpace()->used();
1.2810 +
1.2811 + // update performance counters - this uses a special version of
1.2812 + // update_counters() that allows the utilization to be passed as a
1.2813 + // parameter, avoiding multiple calls to used().
1.2814 + //
1.2815 + _cmsGen->update_counters(cms_used);
1.2816 +
1.2817 + if (CMSIncrementalMode) {
1.2818 + icms_update_allocation_limits();
1.2819 + }
1.2820 +
1.2821 + bitMapLock()->unlock();
1.2822 + releaseFreelistLocks();
1.2823 +
1.2824 + if (!CleanChunkPoolAsync) {
1.2825 + Chunk::clean_chunk_pool();
1.2826 + }
1.2827 +
1.2828 + set_did_compact(false);
1.2829 + _between_prologue_and_epilogue = false; // ready for next cycle
1.2830 +}
1.2831 +
1.2832 +void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
1.2833 + collector()->gc_epilogue(full);
1.2834 +
1.2835 + // Also reset promotion tracking in par gc thread states.
1.2836 + if (CollectedHeap::use_parallel_gc_threads()) {
1.2837 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.2838 + _par_gc_thread_states[i]->promo.stopTrackingPromotions(i);
1.2839 + }
1.2840 + }
1.2841 +}
1.2842 +
1.2843 +void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) {
1.2844 + assert(!incremental_collection_failed(), "Should have been cleared");
1.2845 + cmsSpace()->setPreconsumptionDirtyCardClosure(NULL);
1.2846 + cmsSpace()->gc_epilogue();
1.2847 + // Print stat counters
1.2848 + NOT_PRODUCT(
1.2849 + assert(_numObjectsAllocated == 0, "check");
1.2850 + assert(_numWordsAllocated == 0, "check");
1.2851 + if (Verbose && PrintGC) {
1.2852 + gclog_or_tty->print("Promoted "SIZE_FORMAT" objects, "
1.2853 + SIZE_FORMAT" bytes",
1.2854 + _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord));
1.2855 + }
1.2856 + _numObjectsPromoted = 0;
1.2857 + _numWordsPromoted = 0;
1.2858 + )
1.2859 +
1.2860 + if (PrintGC && Verbose) {
1.2861 + // Call down the chain in contiguous_available needs the freelistLock
1.2862 + // so print this out before releasing the freeListLock.
1.2863 + gclog_or_tty->print(" Contiguous available "SIZE_FORMAT" bytes ",
1.2864 + contiguous_available());
1.2865 + }
1.2866 +}
1.2867 +
1.2868 +#ifndef PRODUCT
1.2869 +bool CMSCollector::have_cms_token() {
1.2870 + Thread* thr = Thread::current();
1.2871 + if (thr->is_VM_thread()) {
1.2872 + return ConcurrentMarkSweepThread::vm_thread_has_cms_token();
1.2873 + } else if (thr->is_ConcurrentGC_thread()) {
1.2874 + return ConcurrentMarkSweepThread::cms_thread_has_cms_token();
1.2875 + } else if (thr->is_GC_task_thread()) {
1.2876 + return ConcurrentMarkSweepThread::vm_thread_has_cms_token() &&
1.2877 + ParGCRareEvent_lock->owned_by_self();
1.2878 + }
1.2879 + return false;
1.2880 +}
1.2881 +#endif
1.2882 +
1.2883 +// Check reachability of the given heap address in CMS generation,
1.2884 +// treating all other generations as roots.
1.2885 +bool CMSCollector::is_cms_reachable(HeapWord* addr) {
1.2886 + // We could "guarantee" below, rather than assert, but i'll
1.2887 + // leave these as "asserts" so that an adventurous debugger
1.2888 + // could try this in the product build provided some subset of
1.2889 + // the conditions were met, provided they were intersted in the
1.2890 + // results and knew that the computation below wouldn't interfere
1.2891 + // with other concurrent computations mutating the structures
1.2892 + // being read or written.
1.2893 + assert(SafepointSynchronize::is_at_safepoint(),
1.2894 + "Else mutations in object graph will make answer suspect");
1.2895 + assert(have_cms_token(), "Should hold cms token");
1.2896 + assert(haveFreelistLocks(), "must hold free list locks");
1.2897 + assert_lock_strong(bitMapLock());
1.2898 +
1.2899 + // Clear the marking bit map array before starting, but, just
1.2900 + // for kicks, first report if the given address is already marked
1.2901 + gclog_or_tty->print_cr("Start: Address 0x%x is%s marked", addr,
1.2902 + _markBitMap.isMarked(addr) ? "" : " not");
1.2903 +
1.2904 + if (verify_after_remark()) {
1.2905 + MutexLockerEx x(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
1.2906 + bool result = verification_mark_bm()->isMarked(addr);
1.2907 + gclog_or_tty->print_cr("TransitiveMark: Address 0x%x %s marked", addr,
1.2908 + result ? "IS" : "is NOT");
1.2909 + return result;
1.2910 + } else {
1.2911 + gclog_or_tty->print_cr("Could not compute result");
1.2912 + return false;
1.2913 + }
1.2914 +}
1.2915 +
1.2916 +
1.2917 +void
1.2918 +CMSCollector::print_on_error(outputStream* st) {
1.2919 + CMSCollector* collector = ConcurrentMarkSweepGeneration::_collector;
1.2920 + if (collector != NULL) {
1.2921 + CMSBitMap* bitmap = &collector->_markBitMap;
1.2922 + st->print_cr("Marking Bits: (CMSBitMap*) " PTR_FORMAT, bitmap);
1.2923 + bitmap->print_on_error(st, " Bits: ");
1.2924 +
1.2925 + st->cr();
1.2926 +
1.2927 + CMSBitMap* mut_bitmap = &collector->_modUnionTable;
1.2928 + st->print_cr("Mod Union Table: (CMSBitMap*) " PTR_FORMAT, mut_bitmap);
1.2929 + mut_bitmap->print_on_error(st, " Bits: ");
1.2930 + }
1.2931 +}
1.2932 +
1.2933 +////////////////////////////////////////////////////////
1.2934 +// CMS Verification Support
1.2935 +////////////////////////////////////////////////////////
1.2936 +// Following the remark phase, the following invariant
1.2937 +// should hold -- each object in the CMS heap which is
1.2938 +// marked in markBitMap() should be marked in the verification_mark_bm().
1.2939 +
1.2940 +class VerifyMarkedClosure: public BitMapClosure {
1.2941 + CMSBitMap* _marks;
1.2942 + bool _failed;
1.2943 +
1.2944 + public:
1.2945 + VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
1.2946 +
1.2947 + bool do_bit(size_t offset) {
1.2948 + HeapWord* addr = _marks->offsetToHeapWord(offset);
1.2949 + if (!_marks->isMarked(addr)) {
1.2950 + oop(addr)->print_on(gclog_or_tty);
1.2951 + gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", addr);
1.2952 + _failed = true;
1.2953 + }
1.2954 + return true;
1.2955 + }
1.2956 +
1.2957 + bool failed() { return _failed; }
1.2958 +};
1.2959 +
1.2960 +bool CMSCollector::verify_after_remark(bool silent) {
1.2961 + if (!silent) gclog_or_tty->print(" [Verifying CMS Marking... ");
1.2962 + MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
1.2963 + static bool init = false;
1.2964 +
1.2965 + assert(SafepointSynchronize::is_at_safepoint(),
1.2966 + "Else mutations in object graph will make answer suspect");
1.2967 + assert(have_cms_token(),
1.2968 + "Else there may be mutual interference in use of "
1.2969 + " verification data structures");
1.2970 + assert(_collectorState > Marking && _collectorState <= Sweeping,
1.2971 + "Else marking info checked here may be obsolete");
1.2972 + assert(haveFreelistLocks(), "must hold free list locks");
1.2973 + assert_lock_strong(bitMapLock());
1.2974 +
1.2975 +
1.2976 + // Allocate marking bit map if not already allocated
1.2977 + if (!init) { // first time
1.2978 + if (!verification_mark_bm()->allocate(_span)) {
1.2979 + return false;
1.2980 + }
1.2981 + init = true;
1.2982 + }
1.2983 +
1.2984 + assert(verification_mark_stack()->isEmpty(), "Should be empty");
1.2985 +
1.2986 + // Turn off refs discovery -- so we will be tracing through refs.
1.2987 + // This is as intended, because by this time
1.2988 + // GC must already have cleared any refs that need to be cleared,
1.2989 + // and traced those that need to be marked; moreover,
1.2990 + // the marking done here is not going to intefere in any
1.2991 + // way with the marking information used by GC.
1.2992 + NoRefDiscovery no_discovery(ref_processor());
1.2993 +
1.2994 + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
1.2995 +
1.2996 + // Clear any marks from a previous round
1.2997 + verification_mark_bm()->clear_all();
1.2998 + assert(verification_mark_stack()->isEmpty(), "markStack should be empty");
1.2999 + verify_work_stacks_empty();
1.3000 +
1.3001 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.3002 + gch->ensure_parsability(false); // fill TLABs, but no need to retire them
1.3003 + // Update the saved marks which may affect the root scans.
1.3004 + gch->save_marks();
1.3005 +
1.3006 + if (CMSRemarkVerifyVariant == 1) {
1.3007 + // In this first variant of verification, we complete
1.3008 + // all marking, then check if the new marks-verctor is
1.3009 + // a subset of the CMS marks-vector.
1.3010 + verify_after_remark_work_1();
1.3011 + } else if (CMSRemarkVerifyVariant == 2) {
1.3012 + // In this second variant of verification, we flag an error
1.3013 + // (i.e. an object reachable in the new marks-vector not reachable
1.3014 + // in the CMS marks-vector) immediately, also indicating the
1.3015 + // identify of an object (A) that references the unmarked object (B) --
1.3016 + // presumably, a mutation to A failed to be picked up by preclean/remark?
1.3017 + verify_after_remark_work_2();
1.3018 + } else {
1.3019 + warning("Unrecognized value %d for CMSRemarkVerifyVariant",
1.3020 + CMSRemarkVerifyVariant);
1.3021 + }
1.3022 + if (!silent) gclog_or_tty->print(" done] ");
1.3023 + return true;
1.3024 +}
1.3025 +
1.3026 +void CMSCollector::verify_after_remark_work_1() {
1.3027 + ResourceMark rm;
1.3028 + HandleMark hm;
1.3029 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.3030 +
1.3031 + // Get a clear set of claim bits for the strong roots processing to work with.
1.3032 + ClassLoaderDataGraph::clear_claimed_marks();
1.3033 +
1.3034 + // Mark from roots one level into CMS
1.3035 + MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
1.3036 + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
1.3037 +
1.3038 + gch->gen_process_strong_roots(_cmsGen->level(),
1.3039 + true, // younger gens are roots
1.3040 + true, // activate StrongRootsScope
1.3041 + false, // not scavenging
1.3042 + SharedHeap::ScanningOption(roots_scanning_options()),
1.3043 + ¬Older,
1.3044 + true, // walk code active on stacks
1.3045 + NULL,
1.3046 + NULL); // SSS: Provide correct closure
1.3047 +
1.3048 + // Now mark from the roots
1.3049 + MarkFromRootsClosure markFromRootsClosure(this, _span,
1.3050 + verification_mark_bm(), verification_mark_stack(),
1.3051 + false /* don't yield */, true /* verifying */);
1.3052 + assert(_restart_addr == NULL, "Expected pre-condition");
1.3053 + verification_mark_bm()->iterate(&markFromRootsClosure);
1.3054 + while (_restart_addr != NULL) {
1.3055 + // Deal with stack overflow: by restarting at the indicated
1.3056 + // address.
1.3057 + HeapWord* ra = _restart_addr;
1.3058 + markFromRootsClosure.reset(ra);
1.3059 + _restart_addr = NULL;
1.3060 + verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
1.3061 + }
1.3062 + assert(verification_mark_stack()->isEmpty(), "Should have been drained");
1.3063 + verify_work_stacks_empty();
1.3064 +
1.3065 + // Marking completed -- now verify that each bit marked in
1.3066 + // verification_mark_bm() is also marked in markBitMap(); flag all
1.3067 + // errors by printing corresponding objects.
1.3068 + VerifyMarkedClosure vcl(markBitMap());
1.3069 + verification_mark_bm()->iterate(&vcl);
1.3070 + if (vcl.failed()) {
1.3071 + gclog_or_tty->print("Verification failed");
1.3072 + Universe::heap()->print_on(gclog_or_tty);
1.3073 + fatal("CMS: failed marking verification after remark");
1.3074 + }
1.3075 +}
1.3076 +
1.3077 +class VerifyKlassOopsKlassClosure : public KlassClosure {
1.3078 + class VerifyKlassOopsClosure : public OopClosure {
1.3079 + CMSBitMap* _bitmap;
1.3080 + public:
1.3081 + VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { }
1.3082 + void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); }
1.3083 + void do_oop(narrowOop* p) { ShouldNotReachHere(); }
1.3084 + } _oop_closure;
1.3085 + public:
1.3086 + VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {}
1.3087 + void do_klass(Klass* k) {
1.3088 + k->oops_do(&_oop_closure);
1.3089 + }
1.3090 +};
1.3091 +
1.3092 +void CMSCollector::verify_after_remark_work_2() {
1.3093 + ResourceMark rm;
1.3094 + HandleMark hm;
1.3095 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.3096 +
1.3097 + // Get a clear set of claim bits for the strong roots processing to work with.
1.3098 + ClassLoaderDataGraph::clear_claimed_marks();
1.3099 +
1.3100 + // Mark from roots one level into CMS
1.3101 + MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(),
1.3102 + markBitMap());
1.3103 + CMKlassClosure klass_closure(¬Older);
1.3104 +
1.3105 + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
1.3106 + gch->gen_process_strong_roots(_cmsGen->level(),
1.3107 + true, // younger gens are roots
1.3108 + true, // activate StrongRootsScope
1.3109 + false, // not scavenging
1.3110 + SharedHeap::ScanningOption(roots_scanning_options()),
1.3111 + ¬Older,
1.3112 + true, // walk code active on stacks
1.3113 + NULL,
1.3114 + &klass_closure);
1.3115 +
1.3116 + // Now mark from the roots
1.3117 + MarkFromRootsVerifyClosure markFromRootsClosure(this, _span,
1.3118 + verification_mark_bm(), markBitMap(), verification_mark_stack());
1.3119 + assert(_restart_addr == NULL, "Expected pre-condition");
1.3120 + verification_mark_bm()->iterate(&markFromRootsClosure);
1.3121 + while (_restart_addr != NULL) {
1.3122 + // Deal with stack overflow: by restarting at the indicated
1.3123 + // address.
1.3124 + HeapWord* ra = _restart_addr;
1.3125 + markFromRootsClosure.reset(ra);
1.3126 + _restart_addr = NULL;
1.3127 + verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
1.3128 + }
1.3129 + assert(verification_mark_stack()->isEmpty(), "Should have been drained");
1.3130 + verify_work_stacks_empty();
1.3131 +
1.3132 + VerifyKlassOopsKlassClosure verify_klass_oops(verification_mark_bm());
1.3133 + ClassLoaderDataGraph::classes_do(&verify_klass_oops);
1.3134 +
1.3135 + // Marking completed -- now verify that each bit marked in
1.3136 + // verification_mark_bm() is also marked in markBitMap(); flag all
1.3137 + // errors by printing corresponding objects.
1.3138 + VerifyMarkedClosure vcl(markBitMap());
1.3139 + verification_mark_bm()->iterate(&vcl);
1.3140 + assert(!vcl.failed(), "Else verification above should not have succeeded");
1.3141 +}
1.3142 +
1.3143 +void ConcurrentMarkSweepGeneration::save_marks() {
1.3144 + // delegate to CMS space
1.3145 + cmsSpace()->save_marks();
1.3146 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.3147 + _par_gc_thread_states[i]->promo.startTrackingPromotions();
1.3148 + }
1.3149 +}
1.3150 +
1.3151 +bool ConcurrentMarkSweepGeneration::no_allocs_since_save_marks() {
1.3152 + return cmsSpace()->no_allocs_since_save_marks();
1.3153 +}
1.3154 +
1.3155 +#define CMS_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \
1.3156 + \
1.3157 +void ConcurrentMarkSweepGeneration:: \
1.3158 +oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \
1.3159 + cl->set_generation(this); \
1.3160 + cmsSpace()->oop_since_save_marks_iterate##nv_suffix(cl); \
1.3161 + cl->reset_generation(); \
1.3162 + save_marks(); \
1.3163 +}
1.3164 +
1.3165 +ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DEFN)
1.3166 +
1.3167 +void
1.3168 +ConcurrentMarkSweepGeneration::younger_refs_iterate(OopsInGenClosure* cl) {
1.3169 + cl->set_generation(this);
1.3170 + younger_refs_in_space_iterate(_cmsSpace, cl);
1.3171 + cl->reset_generation();
1.3172 +}
1.3173 +
1.3174 +void
1.3175 +ConcurrentMarkSweepGeneration::oop_iterate(MemRegion mr, ExtendedOopClosure* cl) {
1.3176 + if (freelistLock()->owned_by_self()) {
1.3177 + Generation::oop_iterate(mr, cl);
1.3178 + } else {
1.3179 + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
1.3180 + Generation::oop_iterate(mr, cl);
1.3181 + }
1.3182 +}
1.3183 +
1.3184 +void
1.3185 +ConcurrentMarkSweepGeneration::oop_iterate(ExtendedOopClosure* cl) {
1.3186 + if (freelistLock()->owned_by_self()) {
1.3187 + Generation::oop_iterate(cl);
1.3188 + } else {
1.3189 + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
1.3190 + Generation::oop_iterate(cl);
1.3191 + }
1.3192 +}
1.3193 +
1.3194 +void
1.3195 +ConcurrentMarkSweepGeneration::object_iterate(ObjectClosure* cl) {
1.3196 + if (freelistLock()->owned_by_self()) {
1.3197 + Generation::object_iterate(cl);
1.3198 + } else {
1.3199 + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
1.3200 + Generation::object_iterate(cl);
1.3201 + }
1.3202 +}
1.3203 +
1.3204 +void
1.3205 +ConcurrentMarkSweepGeneration::safe_object_iterate(ObjectClosure* cl) {
1.3206 + if (freelistLock()->owned_by_self()) {
1.3207 + Generation::safe_object_iterate(cl);
1.3208 + } else {
1.3209 + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
1.3210 + Generation::safe_object_iterate(cl);
1.3211 + }
1.3212 +}
1.3213 +
1.3214 +void
1.3215 +ConcurrentMarkSweepGeneration::post_compact() {
1.3216 +}
1.3217 +
1.3218 +void
1.3219 +ConcurrentMarkSweepGeneration::prepare_for_verify() {
1.3220 + // Fix the linear allocation blocks to look like free blocks.
1.3221 +
1.3222 + // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those
1.3223 + // are not called when the heap is verified during universe initialization and
1.3224 + // at vm shutdown.
1.3225 + if (freelistLock()->owned_by_self()) {
1.3226 + cmsSpace()->prepare_for_verify();
1.3227 + } else {
1.3228 + MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag);
1.3229 + cmsSpace()->prepare_for_verify();
1.3230 + }
1.3231 +}
1.3232 +
1.3233 +void
1.3234 +ConcurrentMarkSweepGeneration::verify() {
1.3235 + // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those
1.3236 + // are not called when the heap is verified during universe initialization and
1.3237 + // at vm shutdown.
1.3238 + if (freelistLock()->owned_by_self()) {
1.3239 + cmsSpace()->verify();
1.3240 + } else {
1.3241 + MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag);
1.3242 + cmsSpace()->verify();
1.3243 + }
1.3244 +}
1.3245 +
1.3246 +void CMSCollector::verify() {
1.3247 + _cmsGen->verify();
1.3248 +}
1.3249 +
1.3250 +#ifndef PRODUCT
1.3251 +bool CMSCollector::overflow_list_is_empty() const {
1.3252 + assert(_num_par_pushes >= 0, "Inconsistency");
1.3253 + if (_overflow_list == NULL) {
1.3254 + assert(_num_par_pushes == 0, "Inconsistency");
1.3255 + }
1.3256 + return _overflow_list == NULL;
1.3257 +}
1.3258 +
1.3259 +// The methods verify_work_stacks_empty() and verify_overflow_empty()
1.3260 +// merely consolidate assertion checks that appear to occur together frequently.
1.3261 +void CMSCollector::verify_work_stacks_empty() const {
1.3262 + assert(_markStack.isEmpty(), "Marking stack should be empty");
1.3263 + assert(overflow_list_is_empty(), "Overflow list should be empty");
1.3264 +}
1.3265 +
1.3266 +void CMSCollector::verify_overflow_empty() const {
1.3267 + assert(overflow_list_is_empty(), "Overflow list should be empty");
1.3268 + assert(no_preserved_marks(), "No preserved marks");
1.3269 +}
1.3270 +#endif // PRODUCT
1.3271 +
1.3272 +// Decide if we want to enable class unloading as part of the
1.3273 +// ensuing concurrent GC cycle. We will collect and
1.3274 +// unload classes if it's the case that:
1.3275 +// (1) an explicit gc request has been made and the flag
1.3276 +// ExplicitGCInvokesConcurrentAndUnloadsClasses is set, OR
1.3277 +// (2) (a) class unloading is enabled at the command line, and
1.3278 +// (b) old gen is getting really full
1.3279 +// NOTE: Provided there is no change in the state of the heap between
1.3280 +// calls to this method, it should have idempotent results. Moreover,
1.3281 +// its results should be monotonically increasing (i.e. going from 0 to 1,
1.3282 +// but not 1 to 0) between successive calls between which the heap was
1.3283 +// not collected. For the implementation below, it must thus rely on
1.3284 +// the property that concurrent_cycles_since_last_unload()
1.3285 +// will not decrease unless a collection cycle happened and that
1.3286 +// _cmsGen->is_too_full() are
1.3287 +// themselves also monotonic in that sense. See check_monotonicity()
1.3288 +// below.
1.3289 +void CMSCollector::update_should_unload_classes() {
1.3290 + _should_unload_classes = false;
1.3291 + // Condition 1 above
1.3292 + if (_full_gc_requested && ExplicitGCInvokesConcurrentAndUnloadsClasses) {
1.3293 + _should_unload_classes = true;
1.3294 + } else if (CMSClassUnloadingEnabled) { // Condition 2.a above
1.3295 + // Disjuncts 2.b.(i,ii,iii) above
1.3296 + _should_unload_classes = (concurrent_cycles_since_last_unload() >=
1.3297 + CMSClassUnloadingMaxInterval)
1.3298 + || _cmsGen->is_too_full();
1.3299 + }
1.3300 +}
1.3301 +
1.3302 +bool ConcurrentMarkSweepGeneration::is_too_full() const {
1.3303 + bool res = should_concurrent_collect();
1.3304 + res = res && (occupancy() > (double)CMSIsTooFullPercentage/100.0);
1.3305 + return res;
1.3306 +}
1.3307 +
1.3308 +void CMSCollector::setup_cms_unloading_and_verification_state() {
1.3309 + const bool should_verify = VerifyBeforeGC || VerifyAfterGC || VerifyDuringGC
1.3310 + || VerifyBeforeExit;
1.3311 + const int rso = SharedHeap::SO_Strings | SharedHeap::SO_CodeCache;
1.3312 +
1.3313 + // We set the proper root for this CMS cycle here.
1.3314 + if (should_unload_classes()) { // Should unload classes this cycle
1.3315 + remove_root_scanning_option(SharedHeap::SO_AllClasses);
1.3316 + add_root_scanning_option(SharedHeap::SO_SystemClasses);
1.3317 + remove_root_scanning_option(rso); // Shrink the root set appropriately
1.3318 + set_verifying(should_verify); // Set verification state for this cycle
1.3319 + return; // Nothing else needs to be done at this time
1.3320 + }
1.3321 +
1.3322 + // Not unloading classes this cycle
1.3323 + assert(!should_unload_classes(), "Inconsitency!");
1.3324 + remove_root_scanning_option(SharedHeap::SO_SystemClasses);
1.3325 + add_root_scanning_option(SharedHeap::SO_AllClasses);
1.3326 +
1.3327 + if ((!verifying() || unloaded_classes_last_cycle()) && should_verify) {
1.3328 + // Include symbols, strings and code cache elements to prevent their resurrection.
1.3329 + add_root_scanning_option(rso);
1.3330 + set_verifying(true);
1.3331 + } else if (verifying() && !should_verify) {
1.3332 + // We were verifying, but some verification flags got disabled.
1.3333 + set_verifying(false);
1.3334 + // Exclude symbols, strings and code cache elements from root scanning to
1.3335 + // reduce IM and RM pauses.
1.3336 + remove_root_scanning_option(rso);
1.3337 + }
1.3338 +}
1.3339 +
1.3340 +
1.3341 +#ifndef PRODUCT
1.3342 +HeapWord* CMSCollector::block_start(const void* p) const {
1.3343 + const HeapWord* addr = (HeapWord*)p;
1.3344 + if (_span.contains(p)) {
1.3345 + if (_cmsGen->cmsSpace()->is_in_reserved(addr)) {
1.3346 + return _cmsGen->cmsSpace()->block_start(p);
1.3347 + }
1.3348 + }
1.3349 + return NULL;
1.3350 +}
1.3351 +#endif
1.3352 +
1.3353 +HeapWord*
1.3354 +ConcurrentMarkSweepGeneration::expand_and_allocate(size_t word_size,
1.3355 + bool tlab,
1.3356 + bool parallel) {
1.3357 + CMSSynchronousYieldRequest yr;
1.3358 + assert(!tlab, "Can't deal with TLAB allocation");
1.3359 + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
1.3360 + expand(word_size*HeapWordSize, MinHeapDeltaBytes,
1.3361 + CMSExpansionCause::_satisfy_allocation);
1.3362 + if (GCExpandToAllocateDelayMillis > 0) {
1.3363 + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
1.3364 + }
1.3365 + return have_lock_and_allocate(word_size, tlab);
1.3366 +}
1.3367 +
1.3368 +// YSR: All of this generation expansion/shrinking stuff is an exact copy of
1.3369 +// OneContigSpaceCardGeneration, which makes me wonder if we should move this
1.3370 +// to CardGeneration and share it...
1.3371 +bool ConcurrentMarkSweepGeneration::expand(size_t bytes, size_t expand_bytes) {
1.3372 + return CardGeneration::expand(bytes, expand_bytes);
1.3373 +}
1.3374 +
1.3375 +void ConcurrentMarkSweepGeneration::expand(size_t bytes, size_t expand_bytes,
1.3376 + CMSExpansionCause::Cause cause)
1.3377 +{
1.3378 +
1.3379 + bool success = expand(bytes, expand_bytes);
1.3380 +
1.3381 + // remember why we expanded; this information is used
1.3382 + // by shouldConcurrentCollect() when making decisions on whether to start
1.3383 + // a new CMS cycle.
1.3384 + if (success) {
1.3385 + set_expansion_cause(cause);
1.3386 + if (PrintGCDetails && Verbose) {
1.3387 + gclog_or_tty->print_cr("Expanded CMS gen for %s",
1.3388 + CMSExpansionCause::to_string(cause));
1.3389 + }
1.3390 + }
1.3391 +}
1.3392 +
1.3393 +HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) {
1.3394 + HeapWord* res = NULL;
1.3395 + MutexLocker x(ParGCRareEvent_lock);
1.3396 + while (true) {
1.3397 + // Expansion by some other thread might make alloc OK now:
1.3398 + res = ps->lab.alloc(word_sz);
1.3399 + if (res != NULL) return res;
1.3400 + // If there's not enough expansion space available, give up.
1.3401 + if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) {
1.3402 + return NULL;
1.3403 + }
1.3404 + // Otherwise, we try expansion.
1.3405 + expand(word_sz*HeapWordSize, MinHeapDeltaBytes,
1.3406 + CMSExpansionCause::_allocate_par_lab);
1.3407 + // Now go around the loop and try alloc again;
1.3408 + // A competing par_promote might beat us to the expansion space,
1.3409 + // so we may go around the loop again if promotion fails agaion.
1.3410 + if (GCExpandToAllocateDelayMillis > 0) {
1.3411 + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
1.3412 + }
1.3413 + }
1.3414 +}
1.3415 +
1.3416 +
1.3417 +bool ConcurrentMarkSweepGeneration::expand_and_ensure_spooling_space(
1.3418 + PromotionInfo* promo) {
1.3419 + MutexLocker x(ParGCRareEvent_lock);
1.3420 + size_t refill_size_bytes = promo->refillSize() * HeapWordSize;
1.3421 + while (true) {
1.3422 + // Expansion by some other thread might make alloc OK now:
1.3423 + if (promo->ensure_spooling_space()) {
1.3424 + assert(promo->has_spooling_space(),
1.3425 + "Post-condition of successful ensure_spooling_space()");
1.3426 + return true;
1.3427 + }
1.3428 + // If there's not enough expansion space available, give up.
1.3429 + if (_virtual_space.uncommitted_size() < refill_size_bytes) {
1.3430 + return false;
1.3431 + }
1.3432 + // Otherwise, we try expansion.
1.3433 + expand(refill_size_bytes, MinHeapDeltaBytes,
1.3434 + CMSExpansionCause::_allocate_par_spooling_space);
1.3435 + // Now go around the loop and try alloc again;
1.3436 + // A competing allocation might beat us to the expansion space,
1.3437 + // so we may go around the loop again if allocation fails again.
1.3438 + if (GCExpandToAllocateDelayMillis > 0) {
1.3439 + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
1.3440 + }
1.3441 + }
1.3442 +}
1.3443 +
1.3444 +
1.3445 +void ConcurrentMarkSweepGeneration::shrink_by(size_t bytes) {
1.3446 + assert_locked_or_safepoint(ExpandHeap_lock);
1.3447 + // Shrink committed space
1.3448 + _virtual_space.shrink_by(bytes);
1.3449 + // Shrink space; this also shrinks the space's BOT
1.3450 + _cmsSpace->set_end((HeapWord*) _virtual_space.high());
1.3451 + size_t new_word_size = heap_word_size(_cmsSpace->capacity());
1.3452 + // Shrink the shared block offset array
1.3453 + _bts->resize(new_word_size);
1.3454 + MemRegion mr(_cmsSpace->bottom(), new_word_size);
1.3455 + // Shrink the card table
1.3456 + Universe::heap()->barrier_set()->resize_covered_region(mr);
1.3457 +
1.3458 + if (Verbose && PrintGC) {
1.3459 + size_t new_mem_size = _virtual_space.committed_size();
1.3460 + size_t old_mem_size = new_mem_size + bytes;
1.3461 + gclog_or_tty->print_cr("Shrinking %s from " SIZE_FORMAT "K to " SIZE_FORMAT "K",
1.3462 + name(), old_mem_size/K, new_mem_size/K);
1.3463 + }
1.3464 +}
1.3465 +
1.3466 +void ConcurrentMarkSweepGeneration::shrink(size_t bytes) {
1.3467 + assert_locked_or_safepoint(Heap_lock);
1.3468 + size_t size = ReservedSpace::page_align_size_down(bytes);
1.3469 + // Only shrink if a compaction was done so that all the free space
1.3470 + // in the generation is in a contiguous block at the end.
1.3471 + if (size > 0 && did_compact()) {
1.3472 + shrink_by(size);
1.3473 + }
1.3474 +}
1.3475 +
1.3476 +bool ConcurrentMarkSweepGeneration::grow_by(size_t bytes) {
1.3477 + assert_locked_or_safepoint(Heap_lock);
1.3478 + bool result = _virtual_space.expand_by(bytes);
1.3479 + if (result) {
1.3480 + size_t new_word_size =
1.3481 + heap_word_size(_virtual_space.committed_size());
1.3482 + MemRegion mr(_cmsSpace->bottom(), new_word_size);
1.3483 + _bts->resize(new_word_size); // resize the block offset shared array
1.3484 + Universe::heap()->barrier_set()->resize_covered_region(mr);
1.3485 + // Hmmmm... why doesn't CFLS::set_end verify locking?
1.3486 + // This is quite ugly; FIX ME XXX
1.3487 + _cmsSpace->assert_locked(freelistLock());
1.3488 + _cmsSpace->set_end((HeapWord*)_virtual_space.high());
1.3489 +
1.3490 + // update the space and generation capacity counters
1.3491 + if (UsePerfData) {
1.3492 + _space_counters->update_capacity();
1.3493 + _gen_counters->update_all();
1.3494 + }
1.3495 +
1.3496 + if (Verbose && PrintGC) {
1.3497 + size_t new_mem_size = _virtual_space.committed_size();
1.3498 + size_t old_mem_size = new_mem_size - bytes;
1.3499 + gclog_or_tty->print_cr("Expanding %s from " SIZE_FORMAT "K by " SIZE_FORMAT "K to " SIZE_FORMAT "K",
1.3500 + name(), old_mem_size/K, bytes/K, new_mem_size/K);
1.3501 + }
1.3502 + }
1.3503 + return result;
1.3504 +}
1.3505 +
1.3506 +bool ConcurrentMarkSweepGeneration::grow_to_reserved() {
1.3507 + assert_locked_or_safepoint(Heap_lock);
1.3508 + bool success = true;
1.3509 + const size_t remaining_bytes = _virtual_space.uncommitted_size();
1.3510 + if (remaining_bytes > 0) {
1.3511 + success = grow_by(remaining_bytes);
1.3512 + DEBUG_ONLY(if (!success) warning("grow to reserved failed");)
1.3513 + }
1.3514 + return success;
1.3515 +}
1.3516 +
1.3517 +void ConcurrentMarkSweepGeneration::shrink_free_list_by(size_t bytes) {
1.3518 + assert_locked_or_safepoint(Heap_lock);
1.3519 + assert_lock_strong(freelistLock());
1.3520 + if (PrintGCDetails && Verbose) {
1.3521 + warning("Shrinking of CMS not yet implemented");
1.3522 + }
1.3523 + return;
1.3524 +}
1.3525 +
1.3526 +
1.3527 +// Simple ctor/dtor wrapper for accounting & timer chores around concurrent
1.3528 +// phases.
1.3529 +class CMSPhaseAccounting: public StackObj {
1.3530 + public:
1.3531 + CMSPhaseAccounting(CMSCollector *collector,
1.3532 + const char *phase,
1.3533 + bool print_cr = true);
1.3534 + ~CMSPhaseAccounting();
1.3535 +
1.3536 + private:
1.3537 + CMSCollector *_collector;
1.3538 + const char *_phase;
1.3539 + elapsedTimer _wallclock;
1.3540 + bool _print_cr;
1.3541 +
1.3542 + public:
1.3543 + // Not MT-safe; so do not pass around these StackObj's
1.3544 + // where they may be accessed by other threads.
1.3545 + jlong wallclock_millis() {
1.3546 + assert(_wallclock.is_active(), "Wall clock should not stop");
1.3547 + _wallclock.stop(); // to record time
1.3548 + jlong ret = _wallclock.milliseconds();
1.3549 + _wallclock.start(); // restart
1.3550 + return ret;
1.3551 + }
1.3552 +};
1.3553 +
1.3554 +CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector,
1.3555 + const char *phase,
1.3556 + bool print_cr) :
1.3557 + _collector(collector), _phase(phase), _print_cr(print_cr) {
1.3558 +
1.3559 + if (PrintCMSStatistics != 0) {
1.3560 + _collector->resetYields();
1.3561 + }
1.3562 + if (PrintGCDetails) {
1.3563 + gclog_or_tty->date_stamp(PrintGCDateStamps);
1.3564 + gclog_or_tty->stamp(PrintGCTimeStamps);
1.3565 + gclog_or_tty->print_cr("[%s-concurrent-%s-start]",
1.3566 + _collector->cmsGen()->short_name(), _phase);
1.3567 + }
1.3568 + _collector->resetTimer();
1.3569 + _wallclock.start();
1.3570 + _collector->startTimer();
1.3571 +}
1.3572 +
1.3573 +CMSPhaseAccounting::~CMSPhaseAccounting() {
1.3574 + assert(_wallclock.is_active(), "Wall clock should not have stopped");
1.3575 + _collector->stopTimer();
1.3576 + _wallclock.stop();
1.3577 + if (PrintGCDetails) {
1.3578 + gclog_or_tty->date_stamp(PrintGCDateStamps);
1.3579 + gclog_or_tty->stamp(PrintGCTimeStamps);
1.3580 + gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]",
1.3581 + _collector->cmsGen()->short_name(),
1.3582 + _phase, _collector->timerValue(), _wallclock.seconds());
1.3583 + if (_print_cr) {
1.3584 + gclog_or_tty->cr();
1.3585 + }
1.3586 + if (PrintCMSStatistics != 0) {
1.3587 + gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase,
1.3588 + _collector->yields());
1.3589 + }
1.3590 + }
1.3591 +}
1.3592 +
1.3593 +// CMS work
1.3594 +
1.3595 +// The common parts of CMSParInitialMarkTask and CMSParRemarkTask.
1.3596 +class CMSParMarkTask : public AbstractGangTask {
1.3597 + protected:
1.3598 + CMSCollector* _collector;
1.3599 + int _n_workers;
1.3600 + CMSParMarkTask(const char* name, CMSCollector* collector, int n_workers) :
1.3601 + AbstractGangTask(name),
1.3602 + _collector(collector),
1.3603 + _n_workers(n_workers) {}
1.3604 + // Work method in support of parallel rescan ... of young gen spaces
1.3605 + void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl,
1.3606 + ContiguousSpace* space,
1.3607 + HeapWord** chunk_array, size_t chunk_top);
1.3608 + void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl);
1.3609 +};
1.3610 +
1.3611 +// Parallel initial mark task
1.3612 +class CMSParInitialMarkTask: public CMSParMarkTask {
1.3613 + public:
1.3614 + CMSParInitialMarkTask(CMSCollector* collector, int n_workers) :
1.3615 + CMSParMarkTask("Scan roots and young gen for initial mark in parallel",
1.3616 + collector, n_workers) {}
1.3617 + void work(uint worker_id);
1.3618 +};
1.3619 +
1.3620 +// Checkpoint the roots into this generation from outside
1.3621 +// this generation. [Note this initial checkpoint need only
1.3622 +// be approximate -- we'll do a catch up phase subsequently.]
1.3623 +void CMSCollector::checkpointRootsInitial(bool asynch) {
1.3624 + assert(_collectorState == InitialMarking, "Wrong collector state");
1.3625 + check_correct_thread_executing();
1.3626 + TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
1.3627 +
1.3628 + save_heap_summary();
1.3629 + report_heap_summary(GCWhen::BeforeGC);
1.3630 +
1.3631 + ReferenceProcessor* rp = ref_processor();
1.3632 + SpecializationStats::clear();
1.3633 + assert(_restart_addr == NULL, "Control point invariant");
1.3634 + if (asynch) {
1.3635 + // acquire locks for subsequent manipulations
1.3636 + MutexLockerEx x(bitMapLock(),
1.3637 + Mutex::_no_safepoint_check_flag);
1.3638 + checkpointRootsInitialWork(asynch);
1.3639 + // enable ("weak") refs discovery
1.3640 + rp->enable_discovery(true /*verify_disabled*/, true /*check_no_refs*/);
1.3641 + _collectorState = Marking;
1.3642 + } else {
1.3643 + // (Weak) Refs discovery: this is controlled from genCollectedHeap::do_collection
1.3644 + // which recognizes if we are a CMS generation, and doesn't try to turn on
1.3645 + // discovery; verify that they aren't meddling.
1.3646 + assert(!rp->discovery_is_atomic(),
1.3647 + "incorrect setting of discovery predicate");
1.3648 + assert(!rp->discovery_enabled(), "genCollectedHeap shouldn't control "
1.3649 + "ref discovery for this generation kind");
1.3650 + // already have locks
1.3651 + checkpointRootsInitialWork(asynch);
1.3652 + // now enable ("weak") refs discovery
1.3653 + rp->enable_discovery(true /*verify_disabled*/, false /*verify_no_refs*/);
1.3654 + _collectorState = Marking;
1.3655 + }
1.3656 + SpecializationStats::print();
1.3657 +}
1.3658 +
1.3659 +void CMSCollector::checkpointRootsInitialWork(bool asynch) {
1.3660 + assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
1.3661 + assert(_collectorState == InitialMarking, "just checking");
1.3662 +
1.3663 + // If there has not been a GC[n-1] since last GC[n] cycle completed,
1.3664 + // precede our marking with a collection of all
1.3665 + // younger generations to keep floating garbage to a minimum.
1.3666 + // XXX: we won't do this for now -- it's an optimization to be done later.
1.3667 +
1.3668 + // already have locks
1.3669 + assert_lock_strong(bitMapLock());
1.3670 + assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
1.3671 +
1.3672 + // Setup the verification and class unloading state for this
1.3673 + // CMS collection cycle.
1.3674 + setup_cms_unloading_and_verification_state();
1.3675 +
1.3676 + NOT_PRODUCT(GCTraceTime t("\ncheckpointRootsInitialWork",
1.3677 + PrintGCDetails && Verbose, true, _gc_timer_cm);)
1.3678 + if (UseAdaptiveSizePolicy) {
1.3679 + size_policy()->checkpoint_roots_initial_begin();
1.3680 + }
1.3681 +
1.3682 + // Reset all the PLAB chunk arrays if necessary.
1.3683 + if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
1.3684 + reset_survivor_plab_arrays();
1.3685 + }
1.3686 +
1.3687 + ResourceMark rm;
1.3688 + HandleMark hm;
1.3689 +
1.3690 + FalseClosure falseClosure;
1.3691 + // In the case of a synchronous collection, we will elide the
1.3692 + // remark step, so it's important to catch all the nmethod oops
1.3693 + // in this step.
1.3694 + // The final 'true' flag to gen_process_strong_roots will ensure this.
1.3695 + // If 'async' is true, we can relax the nmethod tracing.
1.3696 + MarkRefsIntoClosure notOlder(_span, &_markBitMap);
1.3697 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.3698 +
1.3699 + verify_work_stacks_empty();
1.3700 + verify_overflow_empty();
1.3701 +
1.3702 + gch->ensure_parsability(false); // fill TLABs, but no need to retire them
1.3703 + // Update the saved marks which may affect the root scans.
1.3704 + gch->save_marks();
1.3705 +
1.3706 + // weak reference processing has not started yet.
1.3707 + ref_processor()->set_enqueuing_is_done(false);
1.3708 +
1.3709 + // Need to remember all newly created CLDs,
1.3710 + // so that we can guarantee that the remark finds them.
1.3711 + ClassLoaderDataGraph::remember_new_clds(true);
1.3712 +
1.3713 + // Whenever a CLD is found, it will be claimed before proceeding to mark
1.3714 + // the klasses. The claimed marks need to be cleared before marking starts.
1.3715 + ClassLoaderDataGraph::clear_claimed_marks();
1.3716 +
1.3717 + if (CMSPrintEdenSurvivorChunks) {
1.3718 + print_eden_and_survivor_chunk_arrays();
1.3719 + }
1.3720 +
1.3721 + {
1.3722 + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
1.3723 + if (CMSParallelInitialMarkEnabled && CollectedHeap::use_parallel_gc_threads()) {
1.3724 + // The parallel version.
1.3725 + FlexibleWorkGang* workers = gch->workers();
1.3726 + assert(workers != NULL, "Need parallel worker threads.");
1.3727 + int n_workers = workers->active_workers();
1.3728 + CMSParInitialMarkTask tsk(this, n_workers);
1.3729 + gch->set_par_threads(n_workers);
1.3730 + initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
1.3731 + if (n_workers > 1) {
1.3732 + GenCollectedHeap::StrongRootsScope srs(gch);
1.3733 + workers->run_task(&tsk);
1.3734 + } else {
1.3735 + GenCollectedHeap::StrongRootsScope srs(gch);
1.3736 + tsk.work(0);
1.3737 + }
1.3738 + gch->set_par_threads(0);
1.3739 + } else {
1.3740 + // The serial version.
1.3741 + CMKlassClosure klass_closure(¬Older);
1.3742 + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
1.3743 + gch->gen_process_strong_roots(_cmsGen->level(),
1.3744 + true, // younger gens are roots
1.3745 + true, // activate StrongRootsScope
1.3746 + false, // not scavenging
1.3747 + SharedHeap::ScanningOption(roots_scanning_options()),
1.3748 + ¬Older,
1.3749 + true, // walk all of code cache if (so & SO_CodeCache)
1.3750 + NULL,
1.3751 + &klass_closure);
1.3752 + }
1.3753 + }
1.3754 +
1.3755 + // Clear mod-union table; it will be dirtied in the prologue of
1.3756 + // CMS generation per each younger generation collection.
1.3757 +
1.3758 + assert(_modUnionTable.isAllClear(),
1.3759 + "Was cleared in most recent final checkpoint phase"
1.3760 + " or no bits are set in the gc_prologue before the start of the next "
1.3761 + "subsequent marking phase.");
1.3762 +
1.3763 + assert(_ct->klass_rem_set()->mod_union_is_clear(), "Must be");
1.3764 +
1.3765 + // Save the end of the used_region of the constituent generations
1.3766 + // to be used to limit the extent of sweep in each generation.
1.3767 + save_sweep_limits();
1.3768 + if (UseAdaptiveSizePolicy) {
1.3769 + size_policy()->checkpoint_roots_initial_end(gch->gc_cause());
1.3770 + }
1.3771 + verify_overflow_empty();
1.3772 +}
1.3773 +
1.3774 +bool CMSCollector::markFromRoots(bool asynch) {
1.3775 + // we might be tempted to assert that:
1.3776 + // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1.3777 + // "inconsistent argument?");
1.3778 + // However that wouldn't be right, because it's possible that
1.3779 + // a safepoint is indeed in progress as a younger generation
1.3780 + // stop-the-world GC happens even as we mark in this generation.
1.3781 + assert(_collectorState == Marking, "inconsistent state?");
1.3782 + check_correct_thread_executing();
1.3783 + verify_overflow_empty();
1.3784 +
1.3785 + bool res;
1.3786 + if (asynch) {
1.3787 +
1.3788 + // Start the timers for adaptive size policy for the concurrent phases
1.3789 + // Do it here so that the foreground MS can use the concurrent
1.3790 + // timer since a foreground MS might has the sweep done concurrently
1.3791 + // or STW.
1.3792 + if (UseAdaptiveSizePolicy) {
1.3793 + size_policy()->concurrent_marking_begin();
1.3794 + }
1.3795 +
1.3796 + // Weak ref discovery note: We may be discovering weak
1.3797 + // refs in this generation concurrent (but interleaved) with
1.3798 + // weak ref discovery by a younger generation collector.
1.3799 +
1.3800 + CMSTokenSyncWithLocks ts(true, bitMapLock());
1.3801 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.3802 + CMSPhaseAccounting pa(this, "mark", !PrintGCDetails);
1.3803 + res = markFromRootsWork(asynch);
1.3804 + if (res) {
1.3805 + _collectorState = Precleaning;
1.3806 + } else { // We failed and a foreground collection wants to take over
1.3807 + assert(_foregroundGCIsActive, "internal state inconsistency");
1.3808 + assert(_restart_addr == NULL, "foreground will restart from scratch");
1.3809 + if (PrintGCDetails) {
1.3810 + gclog_or_tty->print_cr("bailing out to foreground collection");
1.3811 + }
1.3812 + }
1.3813 + if (UseAdaptiveSizePolicy) {
1.3814 + size_policy()->concurrent_marking_end();
1.3815 + }
1.3816 + } else {
1.3817 + assert(SafepointSynchronize::is_at_safepoint(),
1.3818 + "inconsistent with asynch == false");
1.3819 + if (UseAdaptiveSizePolicy) {
1.3820 + size_policy()->ms_collection_marking_begin();
1.3821 + }
1.3822 + // already have locks
1.3823 + res = markFromRootsWork(asynch);
1.3824 + _collectorState = FinalMarking;
1.3825 + if (UseAdaptiveSizePolicy) {
1.3826 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.3827 + size_policy()->ms_collection_marking_end(gch->gc_cause());
1.3828 + }
1.3829 + }
1.3830 + verify_overflow_empty();
1.3831 + return res;
1.3832 +}
1.3833 +
1.3834 +bool CMSCollector::markFromRootsWork(bool asynch) {
1.3835 + // iterate over marked bits in bit map, doing a full scan and mark
1.3836 + // from these roots using the following algorithm:
1.3837 + // . if oop is to the right of the current scan pointer,
1.3838 + // mark corresponding bit (we'll process it later)
1.3839 + // . else (oop is to left of current scan pointer)
1.3840 + // push oop on marking stack
1.3841 + // . drain the marking stack
1.3842 +
1.3843 + // Note that when we do a marking step we need to hold the
1.3844 + // bit map lock -- recall that direct allocation (by mutators)
1.3845 + // and promotion (by younger generation collectors) is also
1.3846 + // marking the bit map. [the so-called allocate live policy.]
1.3847 + // Because the implementation of bit map marking is not
1.3848 + // robust wrt simultaneous marking of bits in the same word,
1.3849 + // we need to make sure that there is no such interference
1.3850 + // between concurrent such updates.
1.3851 +
1.3852 + // already have locks
1.3853 + assert_lock_strong(bitMapLock());
1.3854 +
1.3855 + verify_work_stacks_empty();
1.3856 + verify_overflow_empty();
1.3857 + bool result = false;
1.3858 + if (CMSConcurrentMTEnabled && ConcGCThreads > 0) {
1.3859 + result = do_marking_mt(asynch);
1.3860 + } else {
1.3861 + result = do_marking_st(asynch);
1.3862 + }
1.3863 + return result;
1.3864 +}
1.3865 +
1.3866 +// Forward decl
1.3867 +class CMSConcMarkingTask;
1.3868 +
1.3869 +class CMSConcMarkingTerminator: public ParallelTaskTerminator {
1.3870 + CMSCollector* _collector;
1.3871 + CMSConcMarkingTask* _task;
1.3872 + public:
1.3873 + virtual void yield();
1.3874 +
1.3875 + // "n_threads" is the number of threads to be terminated.
1.3876 + // "queue_set" is a set of work queues of other threads.
1.3877 + // "collector" is the CMS collector associated with this task terminator.
1.3878 + // "yield" indicates whether we need the gang as a whole to yield.
1.3879 + CMSConcMarkingTerminator(int n_threads, TaskQueueSetSuper* queue_set, CMSCollector* collector) :
1.3880 + ParallelTaskTerminator(n_threads, queue_set),
1.3881 + _collector(collector) { }
1.3882 +
1.3883 + void set_task(CMSConcMarkingTask* task) {
1.3884 + _task = task;
1.3885 + }
1.3886 +};
1.3887 +
1.3888 +class CMSConcMarkingTerminatorTerminator: public TerminatorTerminator {
1.3889 + CMSConcMarkingTask* _task;
1.3890 + public:
1.3891 + bool should_exit_termination();
1.3892 + void set_task(CMSConcMarkingTask* task) {
1.3893 + _task = task;
1.3894 + }
1.3895 +};
1.3896 +
1.3897 +// MT Concurrent Marking Task
1.3898 +class CMSConcMarkingTask: public YieldingFlexibleGangTask {
1.3899 + CMSCollector* _collector;
1.3900 + int _n_workers; // requested/desired # workers
1.3901 + bool _asynch;
1.3902 + bool _result;
1.3903 + CompactibleFreeListSpace* _cms_space;
1.3904 + char _pad_front[64]; // padding to ...
1.3905 + HeapWord* _global_finger; // ... avoid sharing cache line
1.3906 + char _pad_back[64];
1.3907 + HeapWord* _restart_addr;
1.3908 +
1.3909 + // Exposed here for yielding support
1.3910 + Mutex* const _bit_map_lock;
1.3911 +
1.3912 + // The per thread work queues, available here for stealing
1.3913 + OopTaskQueueSet* _task_queues;
1.3914 +
1.3915 + // Termination (and yielding) support
1.3916 + CMSConcMarkingTerminator _term;
1.3917 + CMSConcMarkingTerminatorTerminator _term_term;
1.3918 +
1.3919 + public:
1.3920 + CMSConcMarkingTask(CMSCollector* collector,
1.3921 + CompactibleFreeListSpace* cms_space,
1.3922 + bool asynch,
1.3923 + YieldingFlexibleWorkGang* workers,
1.3924 + OopTaskQueueSet* task_queues):
1.3925 + YieldingFlexibleGangTask("Concurrent marking done multi-threaded"),
1.3926 + _collector(collector),
1.3927 + _cms_space(cms_space),
1.3928 + _asynch(asynch), _n_workers(0), _result(true),
1.3929 + _task_queues(task_queues),
1.3930 + _term(_n_workers, task_queues, _collector),
1.3931 + _bit_map_lock(collector->bitMapLock())
1.3932 + {
1.3933 + _requested_size = _n_workers;
1.3934 + _term.set_task(this);
1.3935 + _term_term.set_task(this);
1.3936 + _restart_addr = _global_finger = _cms_space->bottom();
1.3937 + }
1.3938 +
1.3939 +
1.3940 + OopTaskQueueSet* task_queues() { return _task_queues; }
1.3941 +
1.3942 + OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
1.3943 +
1.3944 + HeapWord** global_finger_addr() { return &_global_finger; }
1.3945 +
1.3946 + CMSConcMarkingTerminator* terminator() { return &_term; }
1.3947 +
1.3948 + virtual void set_for_termination(int active_workers) {
1.3949 + terminator()->reset_for_reuse(active_workers);
1.3950 + }
1.3951 +
1.3952 + void work(uint worker_id);
1.3953 + bool should_yield() {
1.3954 + return ConcurrentMarkSweepThread::should_yield()
1.3955 + && !_collector->foregroundGCIsActive()
1.3956 + && _asynch;
1.3957 + }
1.3958 +
1.3959 + virtual void coordinator_yield(); // stuff done by coordinator
1.3960 + bool result() { return _result; }
1.3961 +
1.3962 + void reset(HeapWord* ra) {
1.3963 + assert(_global_finger >= _cms_space->end(), "Postcondition of ::work(i)");
1.3964 + _restart_addr = _global_finger = ra;
1.3965 + _term.reset_for_reuse();
1.3966 + }
1.3967 +
1.3968 + static bool get_work_from_overflow_stack(CMSMarkStack* ovflw_stk,
1.3969 + OopTaskQueue* work_q);
1.3970 +
1.3971 + private:
1.3972 + void do_scan_and_mark(int i, CompactibleFreeListSpace* sp);
1.3973 + void do_work_steal(int i);
1.3974 + void bump_global_finger(HeapWord* f);
1.3975 +};
1.3976 +
1.3977 +bool CMSConcMarkingTerminatorTerminator::should_exit_termination() {
1.3978 + assert(_task != NULL, "Error");
1.3979 + return _task->yielding();
1.3980 + // Note that we do not need the disjunct || _task->should_yield() above
1.3981 + // because we want terminating threads to yield only if the task
1.3982 + // is already in the midst of yielding, which happens only after at least one
1.3983 + // thread has yielded.
1.3984 +}
1.3985 +
1.3986 +void CMSConcMarkingTerminator::yield() {
1.3987 + if (_task->should_yield()) {
1.3988 + _task->yield();
1.3989 + } else {
1.3990 + ParallelTaskTerminator::yield();
1.3991 + }
1.3992 +}
1.3993 +
1.3994 +////////////////////////////////////////////////////////////////
1.3995 +// Concurrent Marking Algorithm Sketch
1.3996 +////////////////////////////////////////////////////////////////
1.3997 +// Until all tasks exhausted (both spaces):
1.3998 +// -- claim next available chunk
1.3999 +// -- bump global finger via CAS
1.4000 +// -- find first object that starts in this chunk
1.4001 +// and start scanning bitmap from that position
1.4002 +// -- scan marked objects for oops
1.4003 +// -- CAS-mark target, and if successful:
1.4004 +// . if target oop is above global finger (volatile read)
1.4005 +// nothing to do
1.4006 +// . if target oop is in chunk and above local finger
1.4007 +// then nothing to do
1.4008 +// . else push on work-queue
1.4009 +// -- Deal with possible overflow issues:
1.4010 +// . local work-queue overflow causes stuff to be pushed on
1.4011 +// global (common) overflow queue
1.4012 +// . always first empty local work queue
1.4013 +// . then get a batch of oops from global work queue if any
1.4014 +// . then do work stealing
1.4015 +// -- When all tasks claimed (both spaces)
1.4016 +// and local work queue empty,
1.4017 +// then in a loop do:
1.4018 +// . check global overflow stack; steal a batch of oops and trace
1.4019 +// . try to steal from other threads oif GOS is empty
1.4020 +// . if neither is available, offer termination
1.4021 +// -- Terminate and return result
1.4022 +//
1.4023 +void CMSConcMarkingTask::work(uint worker_id) {
1.4024 + elapsedTimer _timer;
1.4025 + ResourceMark rm;
1.4026 + HandleMark hm;
1.4027 +
1.4028 + DEBUG_ONLY(_collector->verify_overflow_empty();)
1.4029 +
1.4030 + // Before we begin work, our work queue should be empty
1.4031 + assert(work_queue(worker_id)->size() == 0, "Expected to be empty");
1.4032 + // Scan the bitmap covering _cms_space, tracing through grey objects.
1.4033 + _timer.start();
1.4034 + do_scan_and_mark(worker_id, _cms_space);
1.4035 + _timer.stop();
1.4036 + if (PrintCMSStatistics != 0) {
1.4037 + gclog_or_tty->print_cr("Finished cms space scanning in %dth thread: %3.3f sec",
1.4038 + worker_id, _timer.seconds());
1.4039 + // XXX: need xxx/xxx type of notation, two timers
1.4040 + }
1.4041 +
1.4042 + // ... do work stealing
1.4043 + _timer.reset();
1.4044 + _timer.start();
1.4045 + do_work_steal(worker_id);
1.4046 + _timer.stop();
1.4047 + if (PrintCMSStatistics != 0) {
1.4048 + gclog_or_tty->print_cr("Finished work stealing in %dth thread: %3.3f sec",
1.4049 + worker_id, _timer.seconds());
1.4050 + // XXX: need xxx/xxx type of notation, two timers
1.4051 + }
1.4052 + assert(_collector->_markStack.isEmpty(), "Should have been emptied");
1.4053 + assert(work_queue(worker_id)->size() == 0, "Should have been emptied");
1.4054 + // Note that under the current task protocol, the
1.4055 + // following assertion is true even of the spaces
1.4056 + // expanded since the completion of the concurrent
1.4057 + // marking. XXX This will likely change under a strict
1.4058 + // ABORT semantics.
1.4059 + // After perm removal the comparison was changed to
1.4060 + // greater than or equal to from strictly greater than.
1.4061 + // Before perm removal the highest address sweep would
1.4062 + // have been at the end of perm gen but now is at the
1.4063 + // end of the tenured gen.
1.4064 + assert(_global_finger >= _cms_space->end(),
1.4065 + "All tasks have been completed");
1.4066 + DEBUG_ONLY(_collector->verify_overflow_empty();)
1.4067 +}
1.4068 +
1.4069 +void CMSConcMarkingTask::bump_global_finger(HeapWord* f) {
1.4070 + HeapWord* read = _global_finger;
1.4071 + HeapWord* cur = read;
1.4072 + while (f > read) {
1.4073 + cur = read;
1.4074 + read = (HeapWord*) Atomic::cmpxchg_ptr(f, &_global_finger, cur);
1.4075 + if (cur == read) {
1.4076 + // our cas succeeded
1.4077 + assert(_global_finger >= f, "protocol consistency");
1.4078 + break;
1.4079 + }
1.4080 + }
1.4081 +}
1.4082 +
1.4083 +// This is really inefficient, and should be redone by
1.4084 +// using (not yet available) block-read and -write interfaces to the
1.4085 +// stack and the work_queue. XXX FIX ME !!!
1.4086 +bool CMSConcMarkingTask::get_work_from_overflow_stack(CMSMarkStack* ovflw_stk,
1.4087 + OopTaskQueue* work_q) {
1.4088 + // Fast lock-free check
1.4089 + if (ovflw_stk->length() == 0) {
1.4090 + return false;
1.4091 + }
1.4092 + assert(work_q->size() == 0, "Shouldn't steal");
1.4093 + MutexLockerEx ml(ovflw_stk->par_lock(),
1.4094 + Mutex::_no_safepoint_check_flag);
1.4095 + // Grab up to 1/4 the size of the work queue
1.4096 + size_t num = MIN2((size_t)(work_q->max_elems() - work_q->size())/4,
1.4097 + (size_t)ParGCDesiredObjsFromOverflowList);
1.4098 + num = MIN2(num, ovflw_stk->length());
1.4099 + for (int i = (int) num; i > 0; i--) {
1.4100 + oop cur = ovflw_stk->pop();
1.4101 + assert(cur != NULL, "Counted wrong?");
1.4102 + work_q->push(cur);
1.4103 + }
1.4104 + return num > 0;
1.4105 +}
1.4106 +
1.4107 +void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
1.4108 + SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
1.4109 + int n_tasks = pst->n_tasks();
1.4110 + // We allow that there may be no tasks to do here because
1.4111 + // we are restarting after a stack overflow.
1.4112 + assert(pst->valid() || n_tasks == 0, "Uninitialized use?");
1.4113 + uint nth_task = 0;
1.4114 +
1.4115 + HeapWord* aligned_start = sp->bottom();
1.4116 + if (sp->used_region().contains(_restart_addr)) {
1.4117 + // Align down to a card boundary for the start of 0th task
1.4118 + // for this space.
1.4119 + aligned_start =
1.4120 + (HeapWord*)align_size_down((uintptr_t)_restart_addr,
1.4121 + CardTableModRefBS::card_size);
1.4122 + }
1.4123 +
1.4124 + size_t chunk_size = sp->marking_task_size();
1.4125 + while (!pst->is_task_claimed(/* reference */ nth_task)) {
1.4126 + // Having claimed the nth task in this space,
1.4127 + // compute the chunk that it corresponds to:
1.4128 + MemRegion span = MemRegion(aligned_start + nth_task*chunk_size,
1.4129 + aligned_start + (nth_task+1)*chunk_size);
1.4130 + // Try and bump the global finger via a CAS;
1.4131 + // note that we need to do the global finger bump
1.4132 + // _before_ taking the intersection below, because
1.4133 + // the task corresponding to that region will be
1.4134 + // deemed done even if the used_region() expands
1.4135 + // because of allocation -- as it almost certainly will
1.4136 + // during start-up while the threads yield in the
1.4137 + // closure below.
1.4138 + HeapWord* finger = span.end();
1.4139 + bump_global_finger(finger); // atomically
1.4140 + // There are null tasks here corresponding to chunks
1.4141 + // beyond the "top" address of the space.
1.4142 + span = span.intersection(sp->used_region());
1.4143 + if (!span.is_empty()) { // Non-null task
1.4144 + HeapWord* prev_obj;
1.4145 + assert(!span.contains(_restart_addr) || nth_task == 0,
1.4146 + "Inconsistency");
1.4147 + if (nth_task == 0) {
1.4148 + // For the 0th task, we'll not need to compute a block_start.
1.4149 + if (span.contains(_restart_addr)) {
1.4150 + // In the case of a restart because of stack overflow,
1.4151 + // we might additionally skip a chunk prefix.
1.4152 + prev_obj = _restart_addr;
1.4153 + } else {
1.4154 + prev_obj = span.start();
1.4155 + }
1.4156 + } else {
1.4157 + // We want to skip the first object because
1.4158 + // the protocol is to scan any object in its entirety
1.4159 + // that _starts_ in this span; a fortiori, any
1.4160 + // object starting in an earlier span is scanned
1.4161 + // as part of an earlier claimed task.
1.4162 + // Below we use the "careful" version of block_start
1.4163 + // so we do not try to navigate uninitialized objects.
1.4164 + prev_obj = sp->block_start_careful(span.start());
1.4165 + // Below we use a variant of block_size that uses the
1.4166 + // Printezis bits to avoid waiting for allocated
1.4167 + // objects to become initialized/parsable.
1.4168 + while (prev_obj < span.start()) {
1.4169 + size_t sz = sp->block_size_no_stall(prev_obj, _collector);
1.4170 + if (sz > 0) {
1.4171 + prev_obj += sz;
1.4172 + } else {
1.4173 + // In this case we may end up doing a bit of redundant
1.4174 + // scanning, but that appears unavoidable, short of
1.4175 + // locking the free list locks; see bug 6324141.
1.4176 + break;
1.4177 + }
1.4178 + }
1.4179 + }
1.4180 + if (prev_obj < span.end()) {
1.4181 + MemRegion my_span = MemRegion(prev_obj, span.end());
1.4182 + // Do the marking work within a non-empty span --
1.4183 + // the last argument to the constructor indicates whether the
1.4184 + // iteration should be incremental with periodic yields.
1.4185 + Par_MarkFromRootsClosure cl(this, _collector, my_span,
1.4186 + &_collector->_markBitMap,
1.4187 + work_queue(i),
1.4188 + &_collector->_markStack,
1.4189 + _asynch);
1.4190 + _collector->_markBitMap.iterate(&cl, my_span.start(), my_span.end());
1.4191 + } // else nothing to do for this task
1.4192 + } // else nothing to do for this task
1.4193 + }
1.4194 + // We'd be tempted to assert here that since there are no
1.4195 + // more tasks left to claim in this space, the global_finger
1.4196 + // must exceed space->top() and a fortiori space->end(). However,
1.4197 + // that would not quite be correct because the bumping of
1.4198 + // global_finger occurs strictly after the claiming of a task,
1.4199 + // so by the time we reach here the global finger may not yet
1.4200 + // have been bumped up by the thread that claimed the last
1.4201 + // task.
1.4202 + pst->all_tasks_completed();
1.4203 +}
1.4204 +
1.4205 +class Par_ConcMarkingClosure: public CMSOopClosure {
1.4206 + private:
1.4207 + CMSCollector* _collector;
1.4208 + CMSConcMarkingTask* _task;
1.4209 + MemRegion _span;
1.4210 + CMSBitMap* _bit_map;
1.4211 + CMSMarkStack* _overflow_stack;
1.4212 + OopTaskQueue* _work_queue;
1.4213 + protected:
1.4214 + DO_OOP_WORK_DEFN
1.4215 + public:
1.4216 + Par_ConcMarkingClosure(CMSCollector* collector, CMSConcMarkingTask* task, OopTaskQueue* work_queue,
1.4217 + CMSBitMap* bit_map, CMSMarkStack* overflow_stack):
1.4218 + CMSOopClosure(collector->ref_processor()),
1.4219 + _collector(collector),
1.4220 + _task(task),
1.4221 + _span(collector->_span),
1.4222 + _work_queue(work_queue),
1.4223 + _bit_map(bit_map),
1.4224 + _overflow_stack(overflow_stack)
1.4225 + { }
1.4226 + virtual void do_oop(oop* p);
1.4227 + virtual void do_oop(narrowOop* p);
1.4228 +
1.4229 + void trim_queue(size_t max);
1.4230 + void handle_stack_overflow(HeapWord* lost);
1.4231 + void do_yield_check() {
1.4232 + if (_task->should_yield()) {
1.4233 + _task->yield();
1.4234 + }
1.4235 + }
1.4236 +};
1.4237 +
1.4238 +// Grey object scanning during work stealing phase --
1.4239 +// the salient assumption here is that any references
1.4240 +// that are in these stolen objects being scanned must
1.4241 +// already have been initialized (else they would not have
1.4242 +// been published), so we do not need to check for
1.4243 +// uninitialized objects before pushing here.
1.4244 +void Par_ConcMarkingClosure::do_oop(oop obj) {
1.4245 + assert(obj->is_oop_or_null(true), "expected an oop or NULL");
1.4246 + HeapWord* addr = (HeapWord*)obj;
1.4247 + // Check if oop points into the CMS generation
1.4248 + // and is not marked
1.4249 + if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
1.4250 + // a white object ...
1.4251 + // If we manage to "claim" the object, by being the
1.4252 + // first thread to mark it, then we push it on our
1.4253 + // marking stack
1.4254 + if (_bit_map->par_mark(addr)) { // ... now grey
1.4255 + // push on work queue (grey set)
1.4256 + bool simulate_overflow = false;
1.4257 + NOT_PRODUCT(
1.4258 + if (CMSMarkStackOverflowALot &&
1.4259 + _collector->simulate_overflow()) {
1.4260 + // simulate a stack overflow
1.4261 + simulate_overflow = true;
1.4262 + }
1.4263 + )
1.4264 + if (simulate_overflow ||
1.4265 + !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
1.4266 + // stack overflow
1.4267 + if (PrintCMSStatistics != 0) {
1.4268 + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
1.4269 + SIZE_FORMAT, _overflow_stack->capacity());
1.4270 + }
1.4271 + // We cannot assert that the overflow stack is full because
1.4272 + // it may have been emptied since.
1.4273 + assert(simulate_overflow ||
1.4274 + _work_queue->size() == _work_queue->max_elems(),
1.4275 + "Else push should have succeeded");
1.4276 + handle_stack_overflow(addr);
1.4277 + }
1.4278 + } // Else, some other thread got there first
1.4279 + do_yield_check();
1.4280 + }
1.4281 +}
1.4282 +
1.4283 +void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
1.4284 +void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
1.4285 +
1.4286 +void Par_ConcMarkingClosure::trim_queue(size_t max) {
1.4287 + while (_work_queue->size() > max) {
1.4288 + oop new_oop;
1.4289 + if (_work_queue->pop_local(new_oop)) {
1.4290 + assert(new_oop->is_oop(), "Should be an oop");
1.4291 + assert(_bit_map->isMarked((HeapWord*)new_oop), "Grey object");
1.4292 + assert(_span.contains((HeapWord*)new_oop), "Not in span");
1.4293 + new_oop->oop_iterate(this); // do_oop() above
1.4294 + do_yield_check();
1.4295 + }
1.4296 + }
1.4297 +}
1.4298 +
1.4299 +// Upon stack overflow, we discard (part of) the stack,
1.4300 +// remembering the least address amongst those discarded
1.4301 +// in CMSCollector's _restart_address.
1.4302 +void Par_ConcMarkingClosure::handle_stack_overflow(HeapWord* lost) {
1.4303 + // We need to do this under a mutex to prevent other
1.4304 + // workers from interfering with the work done below.
1.4305 + MutexLockerEx ml(_overflow_stack->par_lock(),
1.4306 + Mutex::_no_safepoint_check_flag);
1.4307 + // Remember the least grey address discarded
1.4308 + HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost);
1.4309 + _collector->lower_restart_addr(ra);
1.4310 + _overflow_stack->reset(); // discard stack contents
1.4311 + _overflow_stack->expand(); // expand the stack if possible
1.4312 +}
1.4313 +
1.4314 +
1.4315 +void CMSConcMarkingTask::do_work_steal(int i) {
1.4316 + OopTaskQueue* work_q = work_queue(i);
1.4317 + oop obj_to_scan;
1.4318 + CMSBitMap* bm = &(_collector->_markBitMap);
1.4319 + CMSMarkStack* ovflw = &(_collector->_markStack);
1.4320 + int* seed = _collector->hash_seed(i);
1.4321 + Par_ConcMarkingClosure cl(_collector, this, work_q, bm, ovflw);
1.4322 + while (true) {
1.4323 + cl.trim_queue(0);
1.4324 + assert(work_q->size() == 0, "Should have been emptied above");
1.4325 + if (get_work_from_overflow_stack(ovflw, work_q)) {
1.4326 + // Can't assert below because the work obtained from the
1.4327 + // overflow stack may already have been stolen from us.
1.4328 + // assert(work_q->size() > 0, "Work from overflow stack");
1.4329 + continue;
1.4330 + } else if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
1.4331 + assert(obj_to_scan->is_oop(), "Should be an oop");
1.4332 + assert(bm->isMarked((HeapWord*)obj_to_scan), "Grey object");
1.4333 + obj_to_scan->oop_iterate(&cl);
1.4334 + } else if (terminator()->offer_termination(&_term_term)) {
1.4335 + assert(work_q->size() == 0, "Impossible!");
1.4336 + break;
1.4337 + } else if (yielding() || should_yield()) {
1.4338 + yield();
1.4339 + }
1.4340 + }
1.4341 +}
1.4342 +
1.4343 +// This is run by the CMS (coordinator) thread.
1.4344 +void CMSConcMarkingTask::coordinator_yield() {
1.4345 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.4346 + "CMS thread should hold CMS token");
1.4347 + // First give up the locks, then yield, then re-lock
1.4348 + // We should probably use a constructor/destructor idiom to
1.4349 + // do this unlock/lock or modify the MutexUnlocker class to
1.4350 + // serve our purpose. XXX
1.4351 + assert_lock_strong(_bit_map_lock);
1.4352 + _bit_map_lock->unlock();
1.4353 + ConcurrentMarkSweepThread::desynchronize(true);
1.4354 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.4355 + _collector->stopTimer();
1.4356 + if (PrintCMSStatistics != 0) {
1.4357 + _collector->incrementYields();
1.4358 + }
1.4359 + _collector->icms_wait();
1.4360 +
1.4361 + // It is possible for whichever thread initiated the yield request
1.4362 + // not to get a chance to wake up and take the bitmap lock between
1.4363 + // this thread releasing it and reacquiring it. So, while the
1.4364 + // should_yield() flag is on, let's sleep for a bit to give the
1.4365 + // other thread a chance to wake up. The limit imposed on the number
1.4366 + // of iterations is defensive, to avoid any unforseen circumstances
1.4367 + // putting us into an infinite loop. Since it's always been this
1.4368 + // (coordinator_yield()) method that was observed to cause the
1.4369 + // problem, we are using a parameter (CMSCoordinatorYieldSleepCount)
1.4370 + // which is by default non-zero. For the other seven methods that
1.4371 + // also perform the yield operation, as are using a different
1.4372 + // parameter (CMSYieldSleepCount) which is by default zero. This way we
1.4373 + // can enable the sleeping for those methods too, if necessary.
1.4374 + // See 6442774.
1.4375 + //
1.4376 + // We really need to reconsider the synchronization between the GC
1.4377 + // thread and the yield-requesting threads in the future and we
1.4378 + // should really use wait/notify, which is the recommended
1.4379 + // way of doing this type of interaction. Additionally, we should
1.4380 + // consolidate the eight methods that do the yield operation and they
1.4381 + // are almost identical into one for better maintenability and
1.4382 + // readability. See 6445193.
1.4383 + //
1.4384 + // Tony 2006.06.29
1.4385 + for (unsigned i = 0; i < CMSCoordinatorYieldSleepCount &&
1.4386 + ConcurrentMarkSweepThread::should_yield() &&
1.4387 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.4388 + os::sleep(Thread::current(), 1, false);
1.4389 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.4390 + }
1.4391 +
1.4392 + ConcurrentMarkSweepThread::synchronize(true);
1.4393 + _bit_map_lock->lock_without_safepoint_check();
1.4394 + _collector->startTimer();
1.4395 +}
1.4396 +
1.4397 +bool CMSCollector::do_marking_mt(bool asynch) {
1.4398 + assert(ConcGCThreads > 0 && conc_workers() != NULL, "precondition");
1.4399 + int num_workers = AdaptiveSizePolicy::calc_active_conc_workers(
1.4400 + conc_workers()->total_workers(),
1.4401 + conc_workers()->active_workers(),
1.4402 + Threads::number_of_non_daemon_threads());
1.4403 + conc_workers()->set_active_workers(num_workers);
1.4404 +
1.4405 + CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
1.4406 +
1.4407 + CMSConcMarkingTask tsk(this,
1.4408 + cms_space,
1.4409 + asynch,
1.4410 + conc_workers(),
1.4411 + task_queues());
1.4412 +
1.4413 + // Since the actual number of workers we get may be different
1.4414 + // from the number we requested above, do we need to do anything different
1.4415 + // below? In particular, may be we need to subclass the SequantialSubTasksDone
1.4416 + // class?? XXX
1.4417 + cms_space ->initialize_sequential_subtasks_for_marking(num_workers);
1.4418 +
1.4419 + // Refs discovery is already non-atomic.
1.4420 + assert(!ref_processor()->discovery_is_atomic(), "Should be non-atomic");
1.4421 + assert(ref_processor()->discovery_is_mt(), "Discovery should be MT");
1.4422 + conc_workers()->start_task(&tsk);
1.4423 + while (tsk.yielded()) {
1.4424 + tsk.coordinator_yield();
1.4425 + conc_workers()->continue_task(&tsk);
1.4426 + }
1.4427 + // If the task was aborted, _restart_addr will be non-NULL
1.4428 + assert(tsk.completed() || _restart_addr != NULL, "Inconsistency");
1.4429 + while (_restart_addr != NULL) {
1.4430 + // XXX For now we do not make use of ABORTED state and have not
1.4431 + // yet implemented the right abort semantics (even in the original
1.4432 + // single-threaded CMS case). That needs some more investigation
1.4433 + // and is deferred for now; see CR# TBF. 07252005YSR. XXX
1.4434 + assert(!CMSAbortSemantics || tsk.aborted(), "Inconsistency");
1.4435 + // If _restart_addr is non-NULL, a marking stack overflow
1.4436 + // occurred; we need to do a fresh marking iteration from the
1.4437 + // indicated restart address.
1.4438 + if (_foregroundGCIsActive && asynch) {
1.4439 + // We may be running into repeated stack overflows, having
1.4440 + // reached the limit of the stack size, while making very
1.4441 + // slow forward progress. It may be best to bail out and
1.4442 + // let the foreground collector do its job.
1.4443 + // Clear _restart_addr, so that foreground GC
1.4444 + // works from scratch. This avoids the headache of
1.4445 + // a "rescan" which would otherwise be needed because
1.4446 + // of the dirty mod union table & card table.
1.4447 + _restart_addr = NULL;
1.4448 + return false;
1.4449 + }
1.4450 + // Adjust the task to restart from _restart_addr
1.4451 + tsk.reset(_restart_addr);
1.4452 + cms_space ->initialize_sequential_subtasks_for_marking(num_workers,
1.4453 + _restart_addr);
1.4454 + _restart_addr = NULL;
1.4455 + // Get the workers going again
1.4456 + conc_workers()->start_task(&tsk);
1.4457 + while (tsk.yielded()) {
1.4458 + tsk.coordinator_yield();
1.4459 + conc_workers()->continue_task(&tsk);
1.4460 + }
1.4461 + }
1.4462 + assert(tsk.completed(), "Inconsistency");
1.4463 + assert(tsk.result() == true, "Inconsistency");
1.4464 + return true;
1.4465 +}
1.4466 +
1.4467 +bool CMSCollector::do_marking_st(bool asynch) {
1.4468 + ResourceMark rm;
1.4469 + HandleMark hm;
1.4470 +
1.4471 + // Temporarily make refs discovery single threaded (non-MT)
1.4472 + ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false);
1.4473 + MarkFromRootsClosure markFromRootsClosure(this, _span, &_markBitMap,
1.4474 + &_markStack, CMSYield && asynch);
1.4475 + // the last argument to iterate indicates whether the iteration
1.4476 + // should be incremental with periodic yields.
1.4477 + _markBitMap.iterate(&markFromRootsClosure);
1.4478 + // If _restart_addr is non-NULL, a marking stack overflow
1.4479 + // occurred; we need to do a fresh iteration from the
1.4480 + // indicated restart address.
1.4481 + while (_restart_addr != NULL) {
1.4482 + if (_foregroundGCIsActive && asynch) {
1.4483 + // We may be running into repeated stack overflows, having
1.4484 + // reached the limit of the stack size, while making very
1.4485 + // slow forward progress. It may be best to bail out and
1.4486 + // let the foreground collector do its job.
1.4487 + // Clear _restart_addr, so that foreground GC
1.4488 + // works from scratch. This avoids the headache of
1.4489 + // a "rescan" which would otherwise be needed because
1.4490 + // of the dirty mod union table & card table.
1.4491 + _restart_addr = NULL;
1.4492 + return false; // indicating failure to complete marking
1.4493 + }
1.4494 + // Deal with stack overflow:
1.4495 + // we restart marking from _restart_addr
1.4496 + HeapWord* ra = _restart_addr;
1.4497 + markFromRootsClosure.reset(ra);
1.4498 + _restart_addr = NULL;
1.4499 + _markBitMap.iterate(&markFromRootsClosure, ra, _span.end());
1.4500 + }
1.4501 + return true;
1.4502 +}
1.4503 +
1.4504 +void CMSCollector::preclean() {
1.4505 + check_correct_thread_executing();
1.4506 + assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
1.4507 + verify_work_stacks_empty();
1.4508 + verify_overflow_empty();
1.4509 + _abort_preclean = false;
1.4510 + if (CMSPrecleaningEnabled) {
1.4511 + if (!CMSEdenChunksRecordAlways) {
1.4512 + _eden_chunk_index = 0;
1.4513 + }
1.4514 + size_t used = get_eden_used();
1.4515 + size_t capacity = get_eden_capacity();
1.4516 + // Don't start sampling unless we will get sufficiently
1.4517 + // many samples.
1.4518 + if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
1.4519 + * CMSScheduleRemarkEdenPenetration)) {
1.4520 + _start_sampling = true;
1.4521 + } else {
1.4522 + _start_sampling = false;
1.4523 + }
1.4524 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.4525 + CMSPhaseAccounting pa(this, "preclean", !PrintGCDetails);
1.4526 + preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
1.4527 + }
1.4528 + CMSTokenSync x(true); // is cms thread
1.4529 + if (CMSPrecleaningEnabled) {
1.4530 + sample_eden();
1.4531 + _collectorState = AbortablePreclean;
1.4532 + } else {
1.4533 + _collectorState = FinalMarking;
1.4534 + }
1.4535 + verify_work_stacks_empty();
1.4536 + verify_overflow_empty();
1.4537 +}
1.4538 +
1.4539 +// Try and schedule the remark such that young gen
1.4540 +// occupancy is CMSScheduleRemarkEdenPenetration %.
1.4541 +void CMSCollector::abortable_preclean() {
1.4542 + check_correct_thread_executing();
1.4543 + assert(CMSPrecleaningEnabled, "Inconsistent control state");
1.4544 + assert(_collectorState == AbortablePreclean, "Inconsistent control state");
1.4545 +
1.4546 + // If Eden's current occupancy is below this threshold,
1.4547 + // immediately schedule the remark; else preclean
1.4548 + // past the next scavenge in an effort to
1.4549 + // schedule the pause as described avove. By choosing
1.4550 + // CMSScheduleRemarkEdenSizeThreshold >= max eden size
1.4551 + // we will never do an actual abortable preclean cycle.
1.4552 + if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) {
1.4553 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.4554 + CMSPhaseAccounting pa(this, "abortable-preclean", !PrintGCDetails);
1.4555 + // We need more smarts in the abortable preclean
1.4556 + // loop below to deal with cases where allocation
1.4557 + // in young gen is very very slow, and our precleaning
1.4558 + // is running a losing race against a horde of
1.4559 + // mutators intent on flooding us with CMS updates
1.4560 + // (dirty cards).
1.4561 + // One, admittedly dumb, strategy is to give up
1.4562 + // after a certain number of abortable precleaning loops
1.4563 + // or after a certain maximum time. We want to make
1.4564 + // this smarter in the next iteration.
1.4565 + // XXX FIX ME!!! YSR
1.4566 + size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0;
1.4567 + while (!(should_abort_preclean() ||
1.4568 + ConcurrentMarkSweepThread::should_terminate())) {
1.4569 + workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2);
1.4570 + cumworkdone += workdone;
1.4571 + loops++;
1.4572 + // Voluntarily terminate abortable preclean phase if we have
1.4573 + // been at it for too long.
1.4574 + if ((CMSMaxAbortablePrecleanLoops != 0) &&
1.4575 + loops >= CMSMaxAbortablePrecleanLoops) {
1.4576 + if (PrintGCDetails) {
1.4577 + gclog_or_tty->print(" CMS: abort preclean due to loops ");
1.4578 + }
1.4579 + break;
1.4580 + }
1.4581 + if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) {
1.4582 + if (PrintGCDetails) {
1.4583 + gclog_or_tty->print(" CMS: abort preclean due to time ");
1.4584 + }
1.4585 + break;
1.4586 + }
1.4587 + // If we are doing little work each iteration, we should
1.4588 + // take a short break.
1.4589 + if (workdone < CMSAbortablePrecleanMinWorkPerIteration) {
1.4590 + // Sleep for some time, waiting for work to accumulate
1.4591 + stopTimer();
1.4592 + cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis);
1.4593 + startTimer();
1.4594 + waited++;
1.4595 + }
1.4596 + }
1.4597 + if (PrintCMSStatistics > 0) {
1.4598 + gclog_or_tty->print(" [%d iterations, %d waits, %d cards)] ",
1.4599 + loops, waited, cumworkdone);
1.4600 + }
1.4601 + }
1.4602 + CMSTokenSync x(true); // is cms thread
1.4603 + if (_collectorState != Idling) {
1.4604 + assert(_collectorState == AbortablePreclean,
1.4605 + "Spontaneous state transition?");
1.4606 + _collectorState = FinalMarking;
1.4607 + } // Else, a foreground collection completed this CMS cycle.
1.4608 + return;
1.4609 +}
1.4610 +
1.4611 +// Respond to an Eden sampling opportunity
1.4612 +void CMSCollector::sample_eden() {
1.4613 + // Make sure a young gc cannot sneak in between our
1.4614 + // reading and recording of a sample.
1.4615 + assert(Thread::current()->is_ConcurrentGC_thread(),
1.4616 + "Only the cms thread may collect Eden samples");
1.4617 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.4618 + "Should collect samples while holding CMS token");
1.4619 + if (!_start_sampling) {
1.4620 + return;
1.4621 + }
1.4622 + // When CMSEdenChunksRecordAlways is true, the eden chunk array
1.4623 + // is populated by the young generation.
1.4624 + if (_eden_chunk_array != NULL && !CMSEdenChunksRecordAlways) {
1.4625 + if (_eden_chunk_index < _eden_chunk_capacity) {
1.4626 + _eden_chunk_array[_eden_chunk_index] = *_top_addr; // take sample
1.4627 + assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
1.4628 + "Unexpected state of Eden");
1.4629 + // We'd like to check that what we just sampled is an oop-start address;
1.4630 + // however, we cannot do that here since the object may not yet have been
1.4631 + // initialized. So we'll instead do the check when we _use_ this sample
1.4632 + // later.
1.4633 + if (_eden_chunk_index == 0 ||
1.4634 + (pointer_delta(_eden_chunk_array[_eden_chunk_index],
1.4635 + _eden_chunk_array[_eden_chunk_index-1])
1.4636 + >= CMSSamplingGrain)) {
1.4637 + _eden_chunk_index++; // commit sample
1.4638 + }
1.4639 + }
1.4640 + }
1.4641 + if ((_collectorState == AbortablePreclean) && !_abort_preclean) {
1.4642 + size_t used = get_eden_used();
1.4643 + size_t capacity = get_eden_capacity();
1.4644 + assert(used <= capacity, "Unexpected state of Eden");
1.4645 + if (used > (capacity/100 * CMSScheduleRemarkEdenPenetration)) {
1.4646 + _abort_preclean = true;
1.4647 + }
1.4648 + }
1.4649 +}
1.4650 +
1.4651 +
1.4652 +size_t CMSCollector::preclean_work(bool clean_refs, bool clean_survivor) {
1.4653 + assert(_collectorState == Precleaning ||
1.4654 + _collectorState == AbortablePreclean, "incorrect state");
1.4655 + ResourceMark rm;
1.4656 + HandleMark hm;
1.4657 +
1.4658 + // Precleaning is currently not MT but the reference processor
1.4659 + // may be set for MT. Disable it temporarily here.
1.4660 + ReferenceProcessor* rp = ref_processor();
1.4661 + ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(rp, false);
1.4662 +
1.4663 + // Do one pass of scrubbing the discovered reference lists
1.4664 + // to remove any reference objects with strongly-reachable
1.4665 + // referents.
1.4666 + if (clean_refs) {
1.4667 + CMSPrecleanRefsYieldClosure yield_cl(this);
1.4668 + assert(rp->span().equals(_span), "Spans should be equal");
1.4669 + CMSKeepAliveClosure keep_alive(this, _span, &_markBitMap,
1.4670 + &_markStack, true /* preclean */);
1.4671 + CMSDrainMarkingStackClosure complete_trace(this,
1.4672 + _span, &_markBitMap, &_markStack,
1.4673 + &keep_alive, true /* preclean */);
1.4674 +
1.4675 + // We don't want this step to interfere with a young
1.4676 + // collection because we don't want to take CPU
1.4677 + // or memory bandwidth away from the young GC threads
1.4678 + // (which may be as many as there are CPUs).
1.4679 + // Note that we don't need to protect ourselves from
1.4680 + // interference with mutators because they can't
1.4681 + // manipulate the discovered reference lists nor affect
1.4682 + // the computed reachability of the referents, the
1.4683 + // only properties manipulated by the precleaning
1.4684 + // of these reference lists.
1.4685 + stopTimer();
1.4686 + CMSTokenSyncWithLocks x(true /* is cms thread */,
1.4687 + bitMapLock());
1.4688 + startTimer();
1.4689 + sample_eden();
1.4690 +
1.4691 + // The following will yield to allow foreground
1.4692 + // collection to proceed promptly. XXX YSR:
1.4693 + // The code in this method may need further
1.4694 + // tweaking for better performance and some restructuring
1.4695 + // for cleaner interfaces.
1.4696 + GCTimer *gc_timer = NULL; // Currently not tracing concurrent phases
1.4697 + rp->preclean_discovered_references(
1.4698 + rp->is_alive_non_header(), &keep_alive, &complete_trace, &yield_cl,
1.4699 + gc_timer);
1.4700 + }
1.4701 +
1.4702 + if (clean_survivor) { // preclean the active survivor space(s)
1.4703 + assert(_young_gen->kind() == Generation::DefNew ||
1.4704 + _young_gen->kind() == Generation::ParNew ||
1.4705 + _young_gen->kind() == Generation::ASParNew,
1.4706 + "incorrect type for cast");
1.4707 + DefNewGeneration* dng = (DefNewGeneration*)_young_gen;
1.4708 + PushAndMarkClosure pam_cl(this, _span, ref_processor(),
1.4709 + &_markBitMap, &_modUnionTable,
1.4710 + &_markStack, true /* precleaning phase */);
1.4711 + stopTimer();
1.4712 + CMSTokenSyncWithLocks ts(true /* is cms thread */,
1.4713 + bitMapLock());
1.4714 + startTimer();
1.4715 + unsigned int before_count =
1.4716 + GenCollectedHeap::heap()->total_collections();
1.4717 + SurvivorSpacePrecleanClosure
1.4718 + sss_cl(this, _span, &_markBitMap, &_markStack,
1.4719 + &pam_cl, before_count, CMSYield);
1.4720 + dng->from()->object_iterate_careful(&sss_cl);
1.4721 + dng->to()->object_iterate_careful(&sss_cl);
1.4722 + }
1.4723 + MarkRefsIntoAndScanClosure
1.4724 + mrias_cl(_span, ref_processor(), &_markBitMap, &_modUnionTable,
1.4725 + &_markStack, this, CMSYield,
1.4726 + true /* precleaning phase */);
1.4727 + // CAUTION: The following closure has persistent state that may need to
1.4728 + // be reset upon a decrease in the sequence of addresses it
1.4729 + // processes.
1.4730 + ScanMarkedObjectsAgainCarefullyClosure
1.4731 + smoac_cl(this, _span,
1.4732 + &_markBitMap, &_markStack, &mrias_cl, CMSYield);
1.4733 +
1.4734 + // Preclean dirty cards in ModUnionTable and CardTable using
1.4735 + // appropriate convergence criterion;
1.4736 + // repeat CMSPrecleanIter times unless we find that
1.4737 + // we are losing.
1.4738 + assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large");
1.4739 + assert(CMSPrecleanNumerator < CMSPrecleanDenominator,
1.4740 + "Bad convergence multiplier");
1.4741 + assert(CMSPrecleanThreshold >= 100,
1.4742 + "Unreasonably low CMSPrecleanThreshold");
1.4743 +
1.4744 + size_t numIter, cumNumCards, lastNumCards, curNumCards;
1.4745 + for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0;
1.4746 + numIter < CMSPrecleanIter;
1.4747 + numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) {
1.4748 + curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl);
1.4749 + if (Verbose && PrintGCDetails) {
1.4750 + gclog_or_tty->print(" (modUnionTable: %d cards)", curNumCards);
1.4751 + }
1.4752 + // Either there are very few dirty cards, so re-mark
1.4753 + // pause will be small anyway, or our pre-cleaning isn't
1.4754 + // that much faster than the rate at which cards are being
1.4755 + // dirtied, so we might as well stop and re-mark since
1.4756 + // precleaning won't improve our re-mark time by much.
1.4757 + if (curNumCards <= CMSPrecleanThreshold ||
1.4758 + (numIter > 0 &&
1.4759 + (curNumCards * CMSPrecleanDenominator >
1.4760 + lastNumCards * CMSPrecleanNumerator))) {
1.4761 + numIter++;
1.4762 + cumNumCards += curNumCards;
1.4763 + break;
1.4764 + }
1.4765 + }
1.4766 +
1.4767 + preclean_klasses(&mrias_cl, _cmsGen->freelistLock());
1.4768 +
1.4769 + curNumCards = preclean_card_table(_cmsGen, &smoac_cl);
1.4770 + cumNumCards += curNumCards;
1.4771 + if (PrintGCDetails && PrintCMSStatistics != 0) {
1.4772 + gclog_or_tty->print_cr(" (cardTable: %d cards, re-scanned %d cards, %d iterations)",
1.4773 + curNumCards, cumNumCards, numIter);
1.4774 + }
1.4775 + return cumNumCards; // as a measure of useful work done
1.4776 +}
1.4777 +
1.4778 +// PRECLEANING NOTES:
1.4779 +// Precleaning involves:
1.4780 +// . reading the bits of the modUnionTable and clearing the set bits.
1.4781 +// . For the cards corresponding to the set bits, we scan the
1.4782 +// objects on those cards. This means we need the free_list_lock
1.4783 +// so that we can safely iterate over the CMS space when scanning
1.4784 +// for oops.
1.4785 +// . When we scan the objects, we'll be both reading and setting
1.4786 +// marks in the marking bit map, so we'll need the marking bit map.
1.4787 +// . For protecting _collector_state transitions, we take the CGC_lock.
1.4788 +// Note that any races in the reading of of card table entries by the
1.4789 +// CMS thread on the one hand and the clearing of those entries by the
1.4790 +// VM thread or the setting of those entries by the mutator threads on the
1.4791 +// other are quite benign. However, for efficiency it makes sense to keep
1.4792 +// the VM thread from racing with the CMS thread while the latter is
1.4793 +// dirty card info to the modUnionTable. We therefore also use the
1.4794 +// CGC_lock to protect the reading of the card table and the mod union
1.4795 +// table by the CM thread.
1.4796 +// . We run concurrently with mutator updates, so scanning
1.4797 +// needs to be done carefully -- we should not try to scan
1.4798 +// potentially uninitialized objects.
1.4799 +//
1.4800 +// Locking strategy: While holding the CGC_lock, we scan over and
1.4801 +// reset a maximal dirty range of the mod union / card tables, then lock
1.4802 +// the free_list_lock and bitmap lock to do a full marking, then
1.4803 +// release these locks; and repeat the cycle. This allows for a
1.4804 +// certain amount of fairness in the sharing of these locks between
1.4805 +// the CMS collector on the one hand, and the VM thread and the
1.4806 +// mutators on the other.
1.4807 +
1.4808 +// NOTE: preclean_mod_union_table() and preclean_card_table()
1.4809 +// further below are largely identical; if you need to modify
1.4810 +// one of these methods, please check the other method too.
1.4811 +
1.4812 +size_t CMSCollector::preclean_mod_union_table(
1.4813 + ConcurrentMarkSweepGeneration* gen,
1.4814 + ScanMarkedObjectsAgainCarefullyClosure* cl) {
1.4815 + verify_work_stacks_empty();
1.4816 + verify_overflow_empty();
1.4817 +
1.4818 + // strategy: starting with the first card, accumulate contiguous
1.4819 + // ranges of dirty cards; clear these cards, then scan the region
1.4820 + // covered by these cards.
1.4821 +
1.4822 + // Since all of the MUT is committed ahead, we can just use
1.4823 + // that, in case the generations expand while we are precleaning.
1.4824 + // It might also be fine to just use the committed part of the
1.4825 + // generation, but we might potentially miss cards when the
1.4826 + // generation is rapidly expanding while we are in the midst
1.4827 + // of precleaning.
1.4828 + HeapWord* startAddr = gen->reserved().start();
1.4829 + HeapWord* endAddr = gen->reserved().end();
1.4830 +
1.4831 + cl->setFreelistLock(gen->freelistLock()); // needed for yielding
1.4832 +
1.4833 + size_t numDirtyCards, cumNumDirtyCards;
1.4834 + HeapWord *nextAddr, *lastAddr;
1.4835 + for (cumNumDirtyCards = numDirtyCards = 0,
1.4836 + nextAddr = lastAddr = startAddr;
1.4837 + nextAddr < endAddr;
1.4838 + nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
1.4839 +
1.4840 + ResourceMark rm;
1.4841 + HandleMark hm;
1.4842 +
1.4843 + MemRegion dirtyRegion;
1.4844 + {
1.4845 + stopTimer();
1.4846 + // Potential yield point
1.4847 + CMSTokenSync ts(true);
1.4848 + startTimer();
1.4849 + sample_eden();
1.4850 + // Get dirty region starting at nextOffset (inclusive),
1.4851 + // simultaneously clearing it.
1.4852 + dirtyRegion =
1.4853 + _modUnionTable.getAndClearMarkedRegion(nextAddr, endAddr);
1.4854 + assert(dirtyRegion.start() >= nextAddr,
1.4855 + "returned region inconsistent?");
1.4856 + }
1.4857 + // Remember where the next search should begin.
1.4858 + // The returned region (if non-empty) is a right open interval,
1.4859 + // so lastOffset is obtained from the right end of that
1.4860 + // interval.
1.4861 + lastAddr = dirtyRegion.end();
1.4862 + // Should do something more transparent and less hacky XXX
1.4863 + numDirtyCards =
1.4864 + _modUnionTable.heapWordDiffToOffsetDiff(dirtyRegion.word_size());
1.4865 +
1.4866 + // We'll scan the cards in the dirty region (with periodic
1.4867 + // yields for foreground GC as needed).
1.4868 + if (!dirtyRegion.is_empty()) {
1.4869 + assert(numDirtyCards > 0, "consistency check");
1.4870 + HeapWord* stop_point = NULL;
1.4871 + stopTimer();
1.4872 + // Potential yield point
1.4873 + CMSTokenSyncWithLocks ts(true, gen->freelistLock(),
1.4874 + bitMapLock());
1.4875 + startTimer();
1.4876 + {
1.4877 + verify_work_stacks_empty();
1.4878 + verify_overflow_empty();
1.4879 + sample_eden();
1.4880 + stop_point =
1.4881 + gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
1.4882 + }
1.4883 + if (stop_point != NULL) {
1.4884 + // The careful iteration stopped early either because it found an
1.4885 + // uninitialized object, or because we were in the midst of an
1.4886 + // "abortable preclean", which should now be aborted. Redirty
1.4887 + // the bits corresponding to the partially-scanned or unscanned
1.4888 + // cards. We'll either restart at the next block boundary or
1.4889 + // abort the preclean.
1.4890 + assert((_collectorState == AbortablePreclean && should_abort_preclean()),
1.4891 + "Should only be AbortablePreclean.");
1.4892 + _modUnionTable.mark_range(MemRegion(stop_point, dirtyRegion.end()));
1.4893 + if (should_abort_preclean()) {
1.4894 + break; // out of preclean loop
1.4895 + } else {
1.4896 + // Compute the next address at which preclean should pick up;
1.4897 + // might need bitMapLock in order to read P-bits.
1.4898 + lastAddr = next_card_start_after_block(stop_point);
1.4899 + }
1.4900 + }
1.4901 + } else {
1.4902 + assert(lastAddr == endAddr, "consistency check");
1.4903 + assert(numDirtyCards == 0, "consistency check");
1.4904 + break;
1.4905 + }
1.4906 + }
1.4907 + verify_work_stacks_empty();
1.4908 + verify_overflow_empty();
1.4909 + return cumNumDirtyCards;
1.4910 +}
1.4911 +
1.4912 +// NOTE: preclean_mod_union_table() above and preclean_card_table()
1.4913 +// below are largely identical; if you need to modify
1.4914 +// one of these methods, please check the other method too.
1.4915 +
1.4916 +size_t CMSCollector::preclean_card_table(ConcurrentMarkSweepGeneration* gen,
1.4917 + ScanMarkedObjectsAgainCarefullyClosure* cl) {
1.4918 + // strategy: it's similar to precleamModUnionTable above, in that
1.4919 + // we accumulate contiguous ranges of dirty cards, mark these cards
1.4920 + // precleaned, then scan the region covered by these cards.
1.4921 + HeapWord* endAddr = (HeapWord*)(gen->_virtual_space.high());
1.4922 + HeapWord* startAddr = (HeapWord*)(gen->_virtual_space.low());
1.4923 +
1.4924 + cl->setFreelistLock(gen->freelistLock()); // needed for yielding
1.4925 +
1.4926 + size_t numDirtyCards, cumNumDirtyCards;
1.4927 + HeapWord *lastAddr, *nextAddr;
1.4928 +
1.4929 + for (cumNumDirtyCards = numDirtyCards = 0,
1.4930 + nextAddr = lastAddr = startAddr;
1.4931 + nextAddr < endAddr;
1.4932 + nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
1.4933 +
1.4934 + ResourceMark rm;
1.4935 + HandleMark hm;
1.4936 +
1.4937 + MemRegion dirtyRegion;
1.4938 + {
1.4939 + // See comments in "Precleaning notes" above on why we
1.4940 + // do this locking. XXX Could the locking overheads be
1.4941 + // too high when dirty cards are sparse? [I don't think so.]
1.4942 + stopTimer();
1.4943 + CMSTokenSync x(true); // is cms thread
1.4944 + startTimer();
1.4945 + sample_eden();
1.4946 + // Get and clear dirty region from card table
1.4947 + dirtyRegion = _ct->ct_bs()->dirty_card_range_after_reset(
1.4948 + MemRegion(nextAddr, endAddr),
1.4949 + true,
1.4950 + CardTableModRefBS::precleaned_card_val());
1.4951 +
1.4952 + assert(dirtyRegion.start() >= nextAddr,
1.4953 + "returned region inconsistent?");
1.4954 + }
1.4955 + lastAddr = dirtyRegion.end();
1.4956 + numDirtyCards =
1.4957 + dirtyRegion.word_size()/CardTableModRefBS::card_size_in_words;
1.4958 +
1.4959 + if (!dirtyRegion.is_empty()) {
1.4960 + stopTimer();
1.4961 + CMSTokenSyncWithLocks ts(true, gen->freelistLock(), bitMapLock());
1.4962 + startTimer();
1.4963 + sample_eden();
1.4964 + verify_work_stacks_empty();
1.4965 + verify_overflow_empty();
1.4966 + HeapWord* stop_point =
1.4967 + gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
1.4968 + if (stop_point != NULL) {
1.4969 + assert((_collectorState == AbortablePreclean && should_abort_preclean()),
1.4970 + "Should only be AbortablePreclean.");
1.4971 + _ct->ct_bs()->invalidate(MemRegion(stop_point, dirtyRegion.end()));
1.4972 + if (should_abort_preclean()) {
1.4973 + break; // out of preclean loop
1.4974 + } else {
1.4975 + // Compute the next address at which preclean should pick up.
1.4976 + lastAddr = next_card_start_after_block(stop_point);
1.4977 + }
1.4978 + }
1.4979 + } else {
1.4980 + break;
1.4981 + }
1.4982 + }
1.4983 + verify_work_stacks_empty();
1.4984 + verify_overflow_empty();
1.4985 + return cumNumDirtyCards;
1.4986 +}
1.4987 +
1.4988 +class PrecleanKlassClosure : public KlassClosure {
1.4989 + CMKlassClosure _cm_klass_closure;
1.4990 + public:
1.4991 + PrecleanKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {}
1.4992 + void do_klass(Klass* k) {
1.4993 + if (k->has_accumulated_modified_oops()) {
1.4994 + k->clear_accumulated_modified_oops();
1.4995 +
1.4996 + _cm_klass_closure.do_klass(k);
1.4997 + }
1.4998 + }
1.4999 +};
1.5000 +
1.5001 +// The freelist lock is needed to prevent asserts, is it really needed?
1.5002 +void CMSCollector::preclean_klasses(MarkRefsIntoAndScanClosure* cl, Mutex* freelistLock) {
1.5003 +
1.5004 + cl->set_freelistLock(freelistLock);
1.5005 +
1.5006 + CMSTokenSyncWithLocks ts(true, freelistLock, bitMapLock());
1.5007 +
1.5008 + // SSS: Add equivalent to ScanMarkedObjectsAgainCarefullyClosure::do_yield_check and should_abort_preclean?
1.5009 + // SSS: We should probably check if precleaning should be aborted, at suitable intervals?
1.5010 + PrecleanKlassClosure preclean_klass_closure(cl);
1.5011 + ClassLoaderDataGraph::classes_do(&preclean_klass_closure);
1.5012 +
1.5013 + verify_work_stacks_empty();
1.5014 + verify_overflow_empty();
1.5015 +}
1.5016 +
1.5017 +void CMSCollector::checkpointRootsFinal(bool asynch,
1.5018 + bool clear_all_soft_refs, bool init_mark_was_synchronous) {
1.5019 + assert(_collectorState == FinalMarking, "incorrect state transition?");
1.5020 + check_correct_thread_executing();
1.5021 + // world is stopped at this checkpoint
1.5022 + assert(SafepointSynchronize::is_at_safepoint(),
1.5023 + "world should be stopped");
1.5024 + TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
1.5025 +
1.5026 + verify_work_stacks_empty();
1.5027 + verify_overflow_empty();
1.5028 +
1.5029 + SpecializationStats::clear();
1.5030 + if (PrintGCDetails) {
1.5031 + gclog_or_tty->print("[YG occupancy: "SIZE_FORMAT" K ("SIZE_FORMAT" K)]",
1.5032 + _young_gen->used() / K,
1.5033 + _young_gen->capacity() / K);
1.5034 + }
1.5035 + if (asynch) {
1.5036 + if (CMSScavengeBeforeRemark) {
1.5037 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5038 + // Temporarily set flag to false, GCH->do_collection will
1.5039 + // expect it to be false and set to true
1.5040 + FlagSetting fl(gch->_is_gc_active, false);
1.5041 + NOT_PRODUCT(GCTraceTime t("Scavenge-Before-Remark",
1.5042 + PrintGCDetails && Verbose, true, _gc_timer_cm);)
1.5043 + int level = _cmsGen->level() - 1;
1.5044 + if (level >= 0) {
1.5045 + gch->do_collection(true, // full (i.e. force, see below)
1.5046 + false, // !clear_all_soft_refs
1.5047 + 0, // size
1.5048 + false, // is_tlab
1.5049 + level // max_level
1.5050 + );
1.5051 + }
1.5052 + }
1.5053 + FreelistLocker x(this);
1.5054 + MutexLockerEx y(bitMapLock(),
1.5055 + Mutex::_no_safepoint_check_flag);
1.5056 + assert(!init_mark_was_synchronous, "but that's impossible!");
1.5057 + checkpointRootsFinalWork(asynch, clear_all_soft_refs, false);
1.5058 + } else {
1.5059 + // already have all the locks
1.5060 + checkpointRootsFinalWork(asynch, clear_all_soft_refs,
1.5061 + init_mark_was_synchronous);
1.5062 + }
1.5063 + verify_work_stacks_empty();
1.5064 + verify_overflow_empty();
1.5065 + SpecializationStats::print();
1.5066 +}
1.5067 +
1.5068 +void CMSCollector::checkpointRootsFinalWork(bool asynch,
1.5069 + bool clear_all_soft_refs, bool init_mark_was_synchronous) {
1.5070 +
1.5071 + NOT_PRODUCT(GCTraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, _gc_timer_cm);)
1.5072 +
1.5073 + assert(haveFreelistLocks(), "must have free list locks");
1.5074 + assert_lock_strong(bitMapLock());
1.5075 +
1.5076 + if (UseAdaptiveSizePolicy) {
1.5077 + size_policy()->checkpoint_roots_final_begin();
1.5078 + }
1.5079 +
1.5080 + ResourceMark rm;
1.5081 + HandleMark hm;
1.5082 +
1.5083 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5084 +
1.5085 + if (should_unload_classes()) {
1.5086 + CodeCache::gc_prologue();
1.5087 + }
1.5088 + assert(haveFreelistLocks(), "must have free list locks");
1.5089 + assert_lock_strong(bitMapLock());
1.5090 +
1.5091 + if (!init_mark_was_synchronous) {
1.5092 + // We might assume that we need not fill TLAB's when
1.5093 + // CMSScavengeBeforeRemark is set, because we may have just done
1.5094 + // a scavenge which would have filled all TLAB's -- and besides
1.5095 + // Eden would be empty. This however may not always be the case --
1.5096 + // for instance although we asked for a scavenge, it may not have
1.5097 + // happened because of a JNI critical section. We probably need
1.5098 + // a policy for deciding whether we can in that case wait until
1.5099 + // the critical section releases and then do the remark following
1.5100 + // the scavenge, and skip it here. In the absence of that policy,
1.5101 + // or of an indication of whether the scavenge did indeed occur,
1.5102 + // we cannot rely on TLAB's having been filled and must do
1.5103 + // so here just in case a scavenge did not happen.
1.5104 + gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
1.5105 + // Update the saved marks which may affect the root scans.
1.5106 + gch->save_marks();
1.5107 +
1.5108 + if (CMSPrintEdenSurvivorChunks) {
1.5109 + print_eden_and_survivor_chunk_arrays();
1.5110 + }
1.5111 +
1.5112 + {
1.5113 + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
1.5114 +
1.5115 + // Note on the role of the mod union table:
1.5116 + // Since the marker in "markFromRoots" marks concurrently with
1.5117 + // mutators, it is possible for some reachable objects not to have been
1.5118 + // scanned. For instance, an only reference to an object A was
1.5119 + // placed in object B after the marker scanned B. Unless B is rescanned,
1.5120 + // A would be collected. Such updates to references in marked objects
1.5121 + // are detected via the mod union table which is the set of all cards
1.5122 + // dirtied since the first checkpoint in this GC cycle and prior to
1.5123 + // the most recent young generation GC, minus those cleaned up by the
1.5124 + // concurrent precleaning.
1.5125 + if (CMSParallelRemarkEnabled && CollectedHeap::use_parallel_gc_threads()) {
1.5126 + GCTraceTime t("Rescan (parallel) ", PrintGCDetails, false, _gc_timer_cm);
1.5127 + do_remark_parallel();
1.5128 + } else {
1.5129 + GCTraceTime t("Rescan (non-parallel) ", PrintGCDetails, false,
1.5130 + _gc_timer_cm);
1.5131 + do_remark_non_parallel();
1.5132 + }
1.5133 + }
1.5134 + } else {
1.5135 + assert(!asynch, "Can't have init_mark_was_synchronous in asynch mode");
1.5136 + // The initial mark was stop-world, so there's no rescanning to
1.5137 + // do; go straight on to the next step below.
1.5138 + }
1.5139 + verify_work_stacks_empty();
1.5140 + verify_overflow_empty();
1.5141 +
1.5142 + {
1.5143 + NOT_PRODUCT(GCTraceTime ts("refProcessingWork", PrintGCDetails, false, _gc_timer_cm);)
1.5144 + refProcessingWork(asynch, clear_all_soft_refs);
1.5145 + }
1.5146 + verify_work_stacks_empty();
1.5147 + verify_overflow_empty();
1.5148 +
1.5149 + if (should_unload_classes()) {
1.5150 + CodeCache::gc_epilogue();
1.5151 + }
1.5152 + JvmtiExport::gc_epilogue();
1.5153 +
1.5154 + // If we encountered any (marking stack / work queue) overflow
1.5155 + // events during the current CMS cycle, take appropriate
1.5156 + // remedial measures, where possible, so as to try and avoid
1.5157 + // recurrence of that condition.
1.5158 + assert(_markStack.isEmpty(), "No grey objects");
1.5159 + size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
1.5160 + _ser_kac_ovflw + _ser_kac_preclean_ovflw;
1.5161 + if (ser_ovflw > 0) {
1.5162 + if (PrintCMSStatistics != 0) {
1.5163 + gclog_or_tty->print_cr("Marking stack overflow (benign) "
1.5164 + "(pmc_pc="SIZE_FORMAT", pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT
1.5165 + ", kac_preclean="SIZE_FORMAT")",
1.5166 + _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw,
1.5167 + _ser_kac_ovflw, _ser_kac_preclean_ovflw);
1.5168 + }
1.5169 + _markStack.expand();
1.5170 + _ser_pmc_remark_ovflw = 0;
1.5171 + _ser_pmc_preclean_ovflw = 0;
1.5172 + _ser_kac_preclean_ovflw = 0;
1.5173 + _ser_kac_ovflw = 0;
1.5174 + }
1.5175 + if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
1.5176 + if (PrintCMSStatistics != 0) {
1.5177 + gclog_or_tty->print_cr("Work queue overflow (benign) "
1.5178 + "(pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT")",
1.5179 + _par_pmc_remark_ovflw, _par_kac_ovflw);
1.5180 + }
1.5181 + _par_pmc_remark_ovflw = 0;
1.5182 + _par_kac_ovflw = 0;
1.5183 + }
1.5184 + if (PrintCMSStatistics != 0) {
1.5185 + if (_markStack._hit_limit > 0) {
1.5186 + gclog_or_tty->print_cr(" (benign) Hit max stack size limit ("SIZE_FORMAT")",
1.5187 + _markStack._hit_limit);
1.5188 + }
1.5189 + if (_markStack._failed_double > 0) {
1.5190 + gclog_or_tty->print_cr(" (benign) Failed stack doubling ("SIZE_FORMAT"),"
1.5191 + " current capacity "SIZE_FORMAT,
1.5192 + _markStack._failed_double,
1.5193 + _markStack.capacity());
1.5194 + }
1.5195 + }
1.5196 + _markStack._hit_limit = 0;
1.5197 + _markStack._failed_double = 0;
1.5198 +
1.5199 + if ((VerifyAfterGC || VerifyDuringGC) &&
1.5200 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.5201 + verify_after_remark();
1.5202 + }
1.5203 +
1.5204 + _gc_tracer_cm->report_object_count_after_gc(&_is_alive_closure);
1.5205 +
1.5206 + // Change under the freelistLocks.
1.5207 + _collectorState = Sweeping;
1.5208 + // Call isAllClear() under bitMapLock
1.5209 + assert(_modUnionTable.isAllClear(),
1.5210 + "Should be clear by end of the final marking");
1.5211 + assert(_ct->klass_rem_set()->mod_union_is_clear(),
1.5212 + "Should be clear by end of the final marking");
1.5213 + if (UseAdaptiveSizePolicy) {
1.5214 + size_policy()->checkpoint_roots_final_end(gch->gc_cause());
1.5215 + }
1.5216 +}
1.5217 +
1.5218 +void CMSParInitialMarkTask::work(uint worker_id) {
1.5219 + elapsedTimer _timer;
1.5220 + ResourceMark rm;
1.5221 + HandleMark hm;
1.5222 +
1.5223 + // ---------- scan from roots --------------
1.5224 + _timer.start();
1.5225 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5226 + Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap));
1.5227 + CMKlassClosure klass_closure(&par_mri_cl);
1.5228 +
1.5229 + // ---------- young gen roots --------------
1.5230 + {
1.5231 + work_on_young_gen_roots(worker_id, &par_mri_cl);
1.5232 + _timer.stop();
1.5233 + if (PrintCMSStatistics != 0) {
1.5234 + gclog_or_tty->print_cr(
1.5235 + "Finished young gen initial mark scan work in %dth thread: %3.3f sec",
1.5236 + worker_id, _timer.seconds());
1.5237 + }
1.5238 + }
1.5239 +
1.5240 + // ---------- remaining roots --------------
1.5241 + _timer.reset();
1.5242 + _timer.start();
1.5243 + gch->gen_process_strong_roots(_collector->_cmsGen->level(),
1.5244 + false, // yg was scanned above
1.5245 + false, // this is parallel code
1.5246 + false, // not scavenging
1.5247 + SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
1.5248 + &par_mri_cl,
1.5249 + true, // walk all of code cache if (so & SO_CodeCache)
1.5250 + NULL,
1.5251 + &klass_closure);
1.5252 + assert(_collector->should_unload_classes()
1.5253 + || (_collector->CMSCollector::roots_scanning_options() & SharedHeap::SO_CodeCache),
1.5254 + "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
1.5255 + _timer.stop();
1.5256 + if (PrintCMSStatistics != 0) {
1.5257 + gclog_or_tty->print_cr(
1.5258 + "Finished remaining root initial mark scan work in %dth thread: %3.3f sec",
1.5259 + worker_id, _timer.seconds());
1.5260 + }
1.5261 +}
1.5262 +
1.5263 +// Parallel remark task
1.5264 +class CMSParRemarkTask: public CMSParMarkTask {
1.5265 + CompactibleFreeListSpace* _cms_space;
1.5266 +
1.5267 + // The per-thread work queues, available here for stealing.
1.5268 + OopTaskQueueSet* _task_queues;
1.5269 + ParallelTaskTerminator _term;
1.5270 +
1.5271 + public:
1.5272 + // A value of 0 passed to n_workers will cause the number of
1.5273 + // workers to be taken from the active workers in the work gang.
1.5274 + CMSParRemarkTask(CMSCollector* collector,
1.5275 + CompactibleFreeListSpace* cms_space,
1.5276 + int n_workers, FlexibleWorkGang* workers,
1.5277 + OopTaskQueueSet* task_queues):
1.5278 + CMSParMarkTask("Rescan roots and grey objects in parallel",
1.5279 + collector, n_workers),
1.5280 + _cms_space(cms_space),
1.5281 + _task_queues(task_queues),
1.5282 + _term(n_workers, task_queues) { }
1.5283 +
1.5284 + OopTaskQueueSet* task_queues() { return _task_queues; }
1.5285 +
1.5286 + OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
1.5287 +
1.5288 + ParallelTaskTerminator* terminator() { return &_term; }
1.5289 + int n_workers() { return _n_workers; }
1.5290 +
1.5291 + void work(uint worker_id);
1.5292 +
1.5293 + private:
1.5294 + // ... of dirty cards in old space
1.5295 + void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i,
1.5296 + Par_MarkRefsIntoAndScanClosure* cl);
1.5297 +
1.5298 + // ... work stealing for the above
1.5299 + void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed);
1.5300 +};
1.5301 +
1.5302 +class RemarkKlassClosure : public KlassClosure {
1.5303 + CMKlassClosure _cm_klass_closure;
1.5304 + public:
1.5305 + RemarkKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {}
1.5306 + void do_klass(Klass* k) {
1.5307 + // Check if we have modified any oops in the Klass during the concurrent marking.
1.5308 + if (k->has_accumulated_modified_oops()) {
1.5309 + k->clear_accumulated_modified_oops();
1.5310 +
1.5311 + // We could have transfered the current modified marks to the accumulated marks,
1.5312 + // like we do with the Card Table to Mod Union Table. But it's not really necessary.
1.5313 + } else if (k->has_modified_oops()) {
1.5314 + // Don't clear anything, this info is needed by the next young collection.
1.5315 + } else {
1.5316 + // No modified oops in the Klass.
1.5317 + return;
1.5318 + }
1.5319 +
1.5320 + // The klass has modified fields, need to scan the klass.
1.5321 + _cm_klass_closure.do_klass(k);
1.5322 + }
1.5323 +};
1.5324 +
1.5325 +void CMSParMarkTask::work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl) {
1.5326 + DefNewGeneration* dng = _collector->_young_gen->as_DefNewGeneration();
1.5327 + EdenSpace* eden_space = dng->eden();
1.5328 + ContiguousSpace* from_space = dng->from();
1.5329 + ContiguousSpace* to_space = dng->to();
1.5330 +
1.5331 + HeapWord** eca = _collector->_eden_chunk_array;
1.5332 + size_t ect = _collector->_eden_chunk_index;
1.5333 + HeapWord** sca = _collector->_survivor_chunk_array;
1.5334 + size_t sct = _collector->_survivor_chunk_index;
1.5335 +
1.5336 + assert(ect <= _collector->_eden_chunk_capacity, "out of bounds");
1.5337 + assert(sct <= _collector->_survivor_chunk_capacity, "out of bounds");
1.5338 +
1.5339 + do_young_space_rescan(worker_id, cl, to_space, NULL, 0);
1.5340 + do_young_space_rescan(worker_id, cl, from_space, sca, sct);
1.5341 + do_young_space_rescan(worker_id, cl, eden_space, eca, ect);
1.5342 +}
1.5343 +
1.5344 +// work_queue(i) is passed to the closure
1.5345 +// Par_MarkRefsIntoAndScanClosure. The "i" parameter
1.5346 +// also is passed to do_dirty_card_rescan_tasks() and to
1.5347 +// do_work_steal() to select the i-th task_queue.
1.5348 +
1.5349 +void CMSParRemarkTask::work(uint worker_id) {
1.5350 + elapsedTimer _timer;
1.5351 + ResourceMark rm;
1.5352 + HandleMark hm;
1.5353 +
1.5354 + // ---------- rescan from roots --------------
1.5355 + _timer.start();
1.5356 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5357 + Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
1.5358 + _collector->_span, _collector->ref_processor(),
1.5359 + &(_collector->_markBitMap),
1.5360 + work_queue(worker_id));
1.5361 +
1.5362 + // Rescan young gen roots first since these are likely
1.5363 + // coarsely partitioned and may, on that account, constitute
1.5364 + // the critical path; thus, it's best to start off that
1.5365 + // work first.
1.5366 + // ---------- young gen roots --------------
1.5367 + {
1.5368 + work_on_young_gen_roots(worker_id, &par_mrias_cl);
1.5369 + _timer.stop();
1.5370 + if (PrintCMSStatistics != 0) {
1.5371 + gclog_or_tty->print_cr(
1.5372 + "Finished young gen rescan work in %dth thread: %3.3f sec",
1.5373 + worker_id, _timer.seconds());
1.5374 + }
1.5375 + }
1.5376 +
1.5377 + // ---------- remaining roots --------------
1.5378 + _timer.reset();
1.5379 + _timer.start();
1.5380 + gch->gen_process_strong_roots(_collector->_cmsGen->level(),
1.5381 + false, // yg was scanned above
1.5382 + false, // this is parallel code
1.5383 + false, // not scavenging
1.5384 + SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
1.5385 + &par_mrias_cl,
1.5386 + true, // walk all of code cache if (so & SO_CodeCache)
1.5387 + NULL,
1.5388 + NULL); // The dirty klasses will be handled below
1.5389 + assert(_collector->should_unload_classes()
1.5390 + || (_collector->CMSCollector::roots_scanning_options() & SharedHeap::SO_CodeCache),
1.5391 + "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
1.5392 + _timer.stop();
1.5393 + if (PrintCMSStatistics != 0) {
1.5394 + gclog_or_tty->print_cr(
1.5395 + "Finished remaining root rescan work in %dth thread: %3.3f sec",
1.5396 + worker_id, _timer.seconds());
1.5397 + }
1.5398 +
1.5399 + // ---------- unhandled CLD scanning ----------
1.5400 + if (worker_id == 0) { // Single threaded at the moment.
1.5401 + _timer.reset();
1.5402 + _timer.start();
1.5403 +
1.5404 + // Scan all new class loader data objects and new dependencies that were
1.5405 + // introduced during concurrent marking.
1.5406 + ResourceMark rm;
1.5407 + GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
1.5408 + for (int i = 0; i < array->length(); i++) {
1.5409 + par_mrias_cl.do_class_loader_data(array->at(i));
1.5410 + }
1.5411 +
1.5412 + // We don't need to keep track of new CLDs anymore.
1.5413 + ClassLoaderDataGraph::remember_new_clds(false);
1.5414 +
1.5415 + _timer.stop();
1.5416 + if (PrintCMSStatistics != 0) {
1.5417 + gclog_or_tty->print_cr(
1.5418 + "Finished unhandled CLD scanning work in %dth thread: %3.3f sec",
1.5419 + worker_id, _timer.seconds());
1.5420 + }
1.5421 + }
1.5422 +
1.5423 + // ---------- dirty klass scanning ----------
1.5424 + if (worker_id == 0) { // Single threaded at the moment.
1.5425 + _timer.reset();
1.5426 + _timer.start();
1.5427 +
1.5428 + // Scan all classes that was dirtied during the concurrent marking phase.
1.5429 + RemarkKlassClosure remark_klass_closure(&par_mrias_cl);
1.5430 + ClassLoaderDataGraph::classes_do(&remark_klass_closure);
1.5431 +
1.5432 + _timer.stop();
1.5433 + if (PrintCMSStatistics != 0) {
1.5434 + gclog_or_tty->print_cr(
1.5435 + "Finished dirty klass scanning work in %dth thread: %3.3f sec",
1.5436 + worker_id, _timer.seconds());
1.5437 + }
1.5438 + }
1.5439 +
1.5440 + // We might have added oops to ClassLoaderData::_handles during the
1.5441 + // concurrent marking phase. These oops point to newly allocated objects
1.5442 + // that are guaranteed to be kept alive. Either by the direct allocation
1.5443 + // code, or when the young collector processes the strong roots. Hence,
1.5444 + // we don't have to revisit the _handles block during the remark phase.
1.5445 +
1.5446 + // ---------- rescan dirty cards ------------
1.5447 + _timer.reset();
1.5448 + _timer.start();
1.5449 +
1.5450 + // Do the rescan tasks for each of the two spaces
1.5451 + // (cms_space) in turn.
1.5452 + // "worker_id" is passed to select the task_queue for "worker_id"
1.5453 + do_dirty_card_rescan_tasks(_cms_space, worker_id, &par_mrias_cl);
1.5454 + _timer.stop();
1.5455 + if (PrintCMSStatistics != 0) {
1.5456 + gclog_or_tty->print_cr(
1.5457 + "Finished dirty card rescan work in %dth thread: %3.3f sec",
1.5458 + worker_id, _timer.seconds());
1.5459 + }
1.5460 +
1.5461 + // ---------- steal work from other threads ...
1.5462 + // ---------- ... and drain overflow list.
1.5463 + _timer.reset();
1.5464 + _timer.start();
1.5465 + do_work_steal(worker_id, &par_mrias_cl, _collector->hash_seed(worker_id));
1.5466 + _timer.stop();
1.5467 + if (PrintCMSStatistics != 0) {
1.5468 + gclog_or_tty->print_cr(
1.5469 + "Finished work stealing in %dth thread: %3.3f sec",
1.5470 + worker_id, _timer.seconds());
1.5471 + }
1.5472 +}
1.5473 +
1.5474 +// Note that parameter "i" is not used.
1.5475 +void
1.5476 +CMSParMarkTask::do_young_space_rescan(uint worker_id,
1.5477 + OopsInGenClosure* cl, ContiguousSpace* space,
1.5478 + HeapWord** chunk_array, size_t chunk_top) {
1.5479 + // Until all tasks completed:
1.5480 + // . claim an unclaimed task
1.5481 + // . compute region boundaries corresponding to task claimed
1.5482 + // using chunk_array
1.5483 + // . par_oop_iterate(cl) over that region
1.5484 +
1.5485 + ResourceMark rm;
1.5486 + HandleMark hm;
1.5487 +
1.5488 + SequentialSubTasksDone* pst = space->par_seq_tasks();
1.5489 +
1.5490 + uint nth_task = 0;
1.5491 + uint n_tasks = pst->n_tasks();
1.5492 +
1.5493 + if (n_tasks > 0) {
1.5494 + assert(pst->valid(), "Uninitialized use?");
1.5495 + HeapWord *start, *end;
1.5496 + while (!pst->is_task_claimed(/* reference */ nth_task)) {
1.5497 + // We claimed task # nth_task; compute its boundaries.
1.5498 + if (chunk_top == 0) { // no samples were taken
1.5499 + assert(nth_task == 0 && n_tasks == 1, "Can have only 1 EdenSpace task");
1.5500 + start = space->bottom();
1.5501 + end = space->top();
1.5502 + } else if (nth_task == 0) {
1.5503 + start = space->bottom();
1.5504 + end = chunk_array[nth_task];
1.5505 + } else if (nth_task < (uint)chunk_top) {
1.5506 + assert(nth_task >= 1, "Control point invariant");
1.5507 + start = chunk_array[nth_task - 1];
1.5508 + end = chunk_array[nth_task];
1.5509 + } else {
1.5510 + assert(nth_task == (uint)chunk_top, "Control point invariant");
1.5511 + start = chunk_array[chunk_top - 1];
1.5512 + end = space->top();
1.5513 + }
1.5514 + MemRegion mr(start, end);
1.5515 + // Verify that mr is in space
1.5516 + assert(mr.is_empty() || space->used_region().contains(mr),
1.5517 + "Should be in space");
1.5518 + // Verify that "start" is an object boundary
1.5519 + assert(mr.is_empty() || oop(mr.start())->is_oop(),
1.5520 + "Should be an oop");
1.5521 + space->par_oop_iterate(mr, cl);
1.5522 + }
1.5523 + pst->all_tasks_completed();
1.5524 + }
1.5525 +}
1.5526 +
1.5527 +void
1.5528 +CMSParRemarkTask::do_dirty_card_rescan_tasks(
1.5529 + CompactibleFreeListSpace* sp, int i,
1.5530 + Par_MarkRefsIntoAndScanClosure* cl) {
1.5531 + // Until all tasks completed:
1.5532 + // . claim an unclaimed task
1.5533 + // . compute region boundaries corresponding to task claimed
1.5534 + // . transfer dirty bits ct->mut for that region
1.5535 + // . apply rescanclosure to dirty mut bits for that region
1.5536 +
1.5537 + ResourceMark rm;
1.5538 + HandleMark hm;
1.5539 +
1.5540 + OopTaskQueue* work_q = work_queue(i);
1.5541 + ModUnionClosure modUnionClosure(&(_collector->_modUnionTable));
1.5542 + // CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION!
1.5543 + // CAUTION: This closure has state that persists across calls to
1.5544 + // the work method dirty_range_iterate_clear() in that it has
1.5545 + // imbedded in it a (subtype of) UpwardsObjectClosure. The
1.5546 + // use of that state in the imbedded UpwardsObjectClosure instance
1.5547 + // assumes that the cards are always iterated (even if in parallel
1.5548 + // by several threads) in monotonically increasing order per each
1.5549 + // thread. This is true of the implementation below which picks
1.5550 + // card ranges (chunks) in monotonically increasing order globally
1.5551 + // and, a-fortiori, in monotonically increasing order per thread
1.5552 + // (the latter order being a subsequence of the former).
1.5553 + // If the work code below is ever reorganized into a more chaotic
1.5554 + // work-partitioning form than the current "sequential tasks"
1.5555 + // paradigm, the use of that persistent state will have to be
1.5556 + // revisited and modified appropriately. See also related
1.5557 + // bug 4756801 work on which should examine this code to make
1.5558 + // sure that the changes there do not run counter to the
1.5559 + // assumptions made here and necessary for correctness and
1.5560 + // efficiency. Note also that this code might yield inefficient
1.5561 + // behaviour in the case of very large objects that span one or
1.5562 + // more work chunks. Such objects would potentially be scanned
1.5563 + // several times redundantly. Work on 4756801 should try and
1.5564 + // address that performance anomaly if at all possible. XXX
1.5565 + MemRegion full_span = _collector->_span;
1.5566 + CMSBitMap* bm = &(_collector->_markBitMap); // shared
1.5567 + MarkFromDirtyCardsClosure
1.5568 + greyRescanClosure(_collector, full_span, // entire span of interest
1.5569 + sp, bm, work_q, cl);
1.5570 +
1.5571 + SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
1.5572 + assert(pst->valid(), "Uninitialized use?");
1.5573 + uint nth_task = 0;
1.5574 + const int alignment = CardTableModRefBS::card_size * BitsPerWord;
1.5575 + MemRegion span = sp->used_region();
1.5576 + HeapWord* start_addr = span.start();
1.5577 + HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(),
1.5578 + alignment);
1.5579 + const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units
1.5580 + assert((HeapWord*)round_to((intptr_t)start_addr, alignment) ==
1.5581 + start_addr, "Check alignment");
1.5582 + assert((size_t)round_to((intptr_t)chunk_size, alignment) ==
1.5583 + chunk_size, "Check alignment");
1.5584 +
1.5585 + while (!pst->is_task_claimed(/* reference */ nth_task)) {
1.5586 + // Having claimed the nth_task, compute corresponding mem-region,
1.5587 + // which is a-fortiori aligned correctly (i.e. at a MUT bopundary).
1.5588 + // The alignment restriction ensures that we do not need any
1.5589 + // synchronization with other gang-workers while setting or
1.5590 + // clearing bits in thus chunk of the MUT.
1.5591 + MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size,
1.5592 + start_addr + (nth_task+1)*chunk_size);
1.5593 + // The last chunk's end might be way beyond end of the
1.5594 + // used region. In that case pull back appropriately.
1.5595 + if (this_span.end() > end_addr) {
1.5596 + this_span.set_end(end_addr);
1.5597 + assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)");
1.5598 + }
1.5599 + // Iterate over the dirty cards covering this chunk, marking them
1.5600 + // precleaned, and setting the corresponding bits in the mod union
1.5601 + // table. Since we have been careful to partition at Card and MUT-word
1.5602 + // boundaries no synchronization is needed between parallel threads.
1.5603 + _collector->_ct->ct_bs()->dirty_card_iterate(this_span,
1.5604 + &modUnionClosure);
1.5605 +
1.5606 + // Having transferred these marks into the modUnionTable,
1.5607 + // rescan the marked objects on the dirty cards in the modUnionTable.
1.5608 + // Even if this is at a synchronous collection, the initial marking
1.5609 + // may have been done during an asynchronous collection so there
1.5610 + // may be dirty bits in the mod-union table.
1.5611 + _collector->_modUnionTable.dirty_range_iterate_clear(
1.5612 + this_span, &greyRescanClosure);
1.5613 + _collector->_modUnionTable.verifyNoOneBitsInRange(
1.5614 + this_span.start(),
1.5615 + this_span.end());
1.5616 + }
1.5617 + pst->all_tasks_completed(); // declare that i am done
1.5618 +}
1.5619 +
1.5620 +// . see if we can share work_queues with ParNew? XXX
1.5621 +void
1.5622 +CMSParRemarkTask::do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl,
1.5623 + int* seed) {
1.5624 + OopTaskQueue* work_q = work_queue(i);
1.5625 + NOT_PRODUCT(int num_steals = 0;)
1.5626 + oop obj_to_scan;
1.5627 + CMSBitMap* bm = &(_collector->_markBitMap);
1.5628 +
1.5629 + while (true) {
1.5630 + // Completely finish any left over work from (an) earlier round(s)
1.5631 + cl->trim_queue(0);
1.5632 + size_t num_from_overflow_list = MIN2((size_t)(work_q->max_elems() - work_q->size())/4,
1.5633 + (size_t)ParGCDesiredObjsFromOverflowList);
1.5634 + // Now check if there's any work in the overflow list
1.5635 + // Passing ParallelGCThreads as the third parameter, no_of_gc_threads,
1.5636 + // only affects the number of attempts made to get work from the
1.5637 + // overflow list and does not affect the number of workers. Just
1.5638 + // pass ParallelGCThreads so this behavior is unchanged.
1.5639 + if (_collector->par_take_from_overflow_list(num_from_overflow_list,
1.5640 + work_q,
1.5641 + ParallelGCThreads)) {
1.5642 + // found something in global overflow list;
1.5643 + // not yet ready to go stealing work from others.
1.5644 + // We'd like to assert(work_q->size() != 0, ...)
1.5645 + // because we just took work from the overflow list,
1.5646 + // but of course we can't since all of that could have
1.5647 + // been already stolen from us.
1.5648 + // "He giveth and He taketh away."
1.5649 + continue;
1.5650 + }
1.5651 + // Verify that we have no work before we resort to stealing
1.5652 + assert(work_q->size() == 0, "Have work, shouldn't steal");
1.5653 + // Try to steal from other queues that have work
1.5654 + if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
1.5655 + NOT_PRODUCT(num_steals++;)
1.5656 + assert(obj_to_scan->is_oop(), "Oops, not an oop!");
1.5657 + assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
1.5658 + // Do scanning work
1.5659 + obj_to_scan->oop_iterate(cl);
1.5660 + // Loop around, finish this work, and try to steal some more
1.5661 + } else if (terminator()->offer_termination()) {
1.5662 + break; // nirvana from the infinite cycle
1.5663 + }
1.5664 + }
1.5665 + NOT_PRODUCT(
1.5666 + if (PrintCMSStatistics != 0) {
1.5667 + gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
1.5668 + }
1.5669 + )
1.5670 + assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
1.5671 + "Else our work is not yet done");
1.5672 +}
1.5673 +
1.5674 +// Record object boundaries in _eden_chunk_array by sampling the eden
1.5675 +// top in the slow-path eden object allocation code path and record
1.5676 +// the boundaries, if CMSEdenChunksRecordAlways is true. If
1.5677 +// CMSEdenChunksRecordAlways is false, we use the other asynchronous
1.5678 +// sampling in sample_eden() that activates during the part of the
1.5679 +// preclean phase.
1.5680 +void CMSCollector::sample_eden_chunk() {
1.5681 + if (CMSEdenChunksRecordAlways && _eden_chunk_array != NULL) {
1.5682 + if (_eden_chunk_lock->try_lock()) {
1.5683 + // Record a sample. This is the critical section. The contents
1.5684 + // of the _eden_chunk_array have to be non-decreasing in the
1.5685 + // address order.
1.5686 + _eden_chunk_array[_eden_chunk_index] = *_top_addr;
1.5687 + assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
1.5688 + "Unexpected state of Eden");
1.5689 + if (_eden_chunk_index == 0 ||
1.5690 + ((_eden_chunk_array[_eden_chunk_index] > _eden_chunk_array[_eden_chunk_index-1]) &&
1.5691 + (pointer_delta(_eden_chunk_array[_eden_chunk_index],
1.5692 + _eden_chunk_array[_eden_chunk_index-1]) >= CMSSamplingGrain))) {
1.5693 + _eden_chunk_index++; // commit sample
1.5694 + }
1.5695 + _eden_chunk_lock->unlock();
1.5696 + }
1.5697 + }
1.5698 +}
1.5699 +
1.5700 +// Return a thread-local PLAB recording array, as appropriate.
1.5701 +void* CMSCollector::get_data_recorder(int thr_num) {
1.5702 + if (_survivor_plab_array != NULL &&
1.5703 + (CMSPLABRecordAlways ||
1.5704 + (_collectorState > Marking && _collectorState < FinalMarking))) {
1.5705 + assert(thr_num < (int)ParallelGCThreads, "thr_num is out of bounds");
1.5706 + ChunkArray* ca = &_survivor_plab_array[thr_num];
1.5707 + ca->reset(); // clear it so that fresh data is recorded
1.5708 + return (void*) ca;
1.5709 + } else {
1.5710 + return NULL;
1.5711 + }
1.5712 +}
1.5713 +
1.5714 +// Reset all the thread-local PLAB recording arrays
1.5715 +void CMSCollector::reset_survivor_plab_arrays() {
1.5716 + for (uint i = 0; i < ParallelGCThreads; i++) {
1.5717 + _survivor_plab_array[i].reset();
1.5718 + }
1.5719 +}
1.5720 +
1.5721 +// Merge the per-thread plab arrays into the global survivor chunk
1.5722 +// array which will provide the partitioning of the survivor space
1.5723 +// for CMS initial scan and rescan.
1.5724 +void CMSCollector::merge_survivor_plab_arrays(ContiguousSpace* surv,
1.5725 + int no_of_gc_threads) {
1.5726 + assert(_survivor_plab_array != NULL, "Error");
1.5727 + assert(_survivor_chunk_array != NULL, "Error");
1.5728 + assert(_collectorState == FinalMarking ||
1.5729 + (CMSParallelInitialMarkEnabled && _collectorState == InitialMarking), "Error");
1.5730 + for (int j = 0; j < no_of_gc_threads; j++) {
1.5731 + _cursor[j] = 0;
1.5732 + }
1.5733 + HeapWord* top = surv->top();
1.5734 + size_t i;
1.5735 + for (i = 0; i < _survivor_chunk_capacity; i++) { // all sca entries
1.5736 + HeapWord* min_val = top; // Higher than any PLAB address
1.5737 + uint min_tid = 0; // position of min_val this round
1.5738 + for (int j = 0; j < no_of_gc_threads; j++) {
1.5739 + ChunkArray* cur_sca = &_survivor_plab_array[j];
1.5740 + if (_cursor[j] == cur_sca->end()) {
1.5741 + continue;
1.5742 + }
1.5743 + assert(_cursor[j] < cur_sca->end(), "ctl pt invariant");
1.5744 + HeapWord* cur_val = cur_sca->nth(_cursor[j]);
1.5745 + assert(surv->used_region().contains(cur_val), "Out of bounds value");
1.5746 + if (cur_val < min_val) {
1.5747 + min_tid = j;
1.5748 + min_val = cur_val;
1.5749 + } else {
1.5750 + assert(cur_val < top, "All recorded addresses should be less");
1.5751 + }
1.5752 + }
1.5753 + // At this point min_val and min_tid are respectively
1.5754 + // the least address in _survivor_plab_array[j]->nth(_cursor[j])
1.5755 + // and the thread (j) that witnesses that address.
1.5756 + // We record this address in the _survivor_chunk_array[i]
1.5757 + // and increment _cursor[min_tid] prior to the next round i.
1.5758 + if (min_val == top) {
1.5759 + break;
1.5760 + }
1.5761 + _survivor_chunk_array[i] = min_val;
1.5762 + _cursor[min_tid]++;
1.5763 + }
1.5764 + // We are all done; record the size of the _survivor_chunk_array
1.5765 + _survivor_chunk_index = i; // exclusive: [0, i)
1.5766 + if (PrintCMSStatistics > 0) {
1.5767 + gclog_or_tty->print(" (Survivor:" SIZE_FORMAT "chunks) ", i);
1.5768 + }
1.5769 + // Verify that we used up all the recorded entries
1.5770 + #ifdef ASSERT
1.5771 + size_t total = 0;
1.5772 + for (int j = 0; j < no_of_gc_threads; j++) {
1.5773 + assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
1.5774 + total += _cursor[j];
1.5775 + }
1.5776 + assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
1.5777 + // Check that the merged array is in sorted order
1.5778 + if (total > 0) {
1.5779 + for (size_t i = 0; i < total - 1; i++) {
1.5780 + if (PrintCMSStatistics > 0) {
1.5781 + gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
1.5782 + i, _survivor_chunk_array[i]);
1.5783 + }
1.5784 + assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
1.5785 + "Not sorted");
1.5786 + }
1.5787 + }
1.5788 + #endif // ASSERT
1.5789 +}
1.5790 +
1.5791 +// Set up the space's par_seq_tasks structure for work claiming
1.5792 +// for parallel initial scan and rescan of young gen.
1.5793 +// See ParRescanTask where this is currently used.
1.5794 +void
1.5795 +CMSCollector::
1.5796 +initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
1.5797 + assert(n_threads > 0, "Unexpected n_threads argument");
1.5798 + DefNewGeneration* dng = (DefNewGeneration*)_young_gen;
1.5799 +
1.5800 + // Eden space
1.5801 + if (!dng->eden()->is_empty()) {
1.5802 + SequentialSubTasksDone* pst = dng->eden()->par_seq_tasks();
1.5803 + assert(!pst->valid(), "Clobbering existing data?");
1.5804 + // Each valid entry in [0, _eden_chunk_index) represents a task.
1.5805 + size_t n_tasks = _eden_chunk_index + 1;
1.5806 + assert(n_tasks == 1 || _eden_chunk_array != NULL, "Error");
1.5807 + // Sets the condition for completion of the subtask (how many threads
1.5808 + // need to finish in order to be done).
1.5809 + pst->set_n_threads(n_threads);
1.5810 + pst->set_n_tasks((int)n_tasks);
1.5811 + }
1.5812 +
1.5813 + // Merge the survivor plab arrays into _survivor_chunk_array
1.5814 + if (_survivor_plab_array != NULL) {
1.5815 + merge_survivor_plab_arrays(dng->from(), n_threads);
1.5816 + } else {
1.5817 + assert(_survivor_chunk_index == 0, "Error");
1.5818 + }
1.5819 +
1.5820 + // To space
1.5821 + {
1.5822 + SequentialSubTasksDone* pst = dng->to()->par_seq_tasks();
1.5823 + assert(!pst->valid(), "Clobbering existing data?");
1.5824 + // Sets the condition for completion of the subtask (how many threads
1.5825 + // need to finish in order to be done).
1.5826 + pst->set_n_threads(n_threads);
1.5827 + pst->set_n_tasks(1);
1.5828 + assert(pst->valid(), "Error");
1.5829 + }
1.5830 +
1.5831 + // From space
1.5832 + {
1.5833 + SequentialSubTasksDone* pst = dng->from()->par_seq_tasks();
1.5834 + assert(!pst->valid(), "Clobbering existing data?");
1.5835 + size_t n_tasks = _survivor_chunk_index + 1;
1.5836 + assert(n_tasks == 1 || _survivor_chunk_array != NULL, "Error");
1.5837 + // Sets the condition for completion of the subtask (how many threads
1.5838 + // need to finish in order to be done).
1.5839 + pst->set_n_threads(n_threads);
1.5840 + pst->set_n_tasks((int)n_tasks);
1.5841 + assert(pst->valid(), "Error");
1.5842 + }
1.5843 +}
1.5844 +
1.5845 +// Parallel version of remark
1.5846 +void CMSCollector::do_remark_parallel() {
1.5847 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5848 + FlexibleWorkGang* workers = gch->workers();
1.5849 + assert(workers != NULL, "Need parallel worker threads.");
1.5850 + // Choose to use the number of GC workers most recently set
1.5851 + // into "active_workers". If active_workers is not set, set it
1.5852 + // to ParallelGCThreads.
1.5853 + int n_workers = workers->active_workers();
1.5854 + if (n_workers == 0) {
1.5855 + assert(n_workers > 0, "Should have been set during scavenge");
1.5856 + n_workers = ParallelGCThreads;
1.5857 + workers->set_active_workers(n_workers);
1.5858 + }
1.5859 + CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
1.5860 +
1.5861 + CMSParRemarkTask tsk(this,
1.5862 + cms_space,
1.5863 + n_workers, workers, task_queues());
1.5864 +
1.5865 + // Set up for parallel process_strong_roots work.
1.5866 + gch->set_par_threads(n_workers);
1.5867 + // We won't be iterating over the cards in the card table updating
1.5868 + // the younger_gen cards, so we shouldn't call the following else
1.5869 + // the verification code as well as subsequent younger_refs_iterate
1.5870 + // code would get confused. XXX
1.5871 + // gch->rem_set()->prepare_for_younger_refs_iterate(true); // parallel
1.5872 +
1.5873 + // The young gen rescan work will not be done as part of
1.5874 + // process_strong_roots (which currently doesn't knw how to
1.5875 + // parallelize such a scan), but rather will be broken up into
1.5876 + // a set of parallel tasks (via the sampling that the [abortable]
1.5877 + // preclean phase did of EdenSpace, plus the [two] tasks of
1.5878 + // scanning the [two] survivor spaces. Further fine-grain
1.5879 + // parallelization of the scanning of the survivor spaces
1.5880 + // themselves, and of precleaning of the younger gen itself
1.5881 + // is deferred to the future.
1.5882 + initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
1.5883 +
1.5884 + // The dirty card rescan work is broken up into a "sequence"
1.5885 + // of parallel tasks (per constituent space) that are dynamically
1.5886 + // claimed by the parallel threads.
1.5887 + cms_space->initialize_sequential_subtasks_for_rescan(n_workers);
1.5888 +
1.5889 + // It turns out that even when we're using 1 thread, doing the work in a
1.5890 + // separate thread causes wide variance in run times. We can't help this
1.5891 + // in the multi-threaded case, but we special-case n=1 here to get
1.5892 + // repeatable measurements of the 1-thread overhead of the parallel code.
1.5893 + if (n_workers > 1) {
1.5894 + // Make refs discovery MT-safe, if it isn't already: it may not
1.5895 + // necessarily be so, since it's possible that we are doing
1.5896 + // ST marking.
1.5897 + ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), true);
1.5898 + GenCollectedHeap::StrongRootsScope srs(gch);
1.5899 + workers->run_task(&tsk);
1.5900 + } else {
1.5901 + ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
1.5902 + GenCollectedHeap::StrongRootsScope srs(gch);
1.5903 + tsk.work(0);
1.5904 + }
1.5905 +
1.5906 + gch->set_par_threads(0); // 0 ==> non-parallel.
1.5907 + // restore, single-threaded for now, any preserved marks
1.5908 + // as a result of work_q overflow
1.5909 + restore_preserved_marks_if_any();
1.5910 +}
1.5911 +
1.5912 +// Non-parallel version of remark
1.5913 +void CMSCollector::do_remark_non_parallel() {
1.5914 + ResourceMark rm;
1.5915 + HandleMark hm;
1.5916 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.5917 + ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
1.5918 +
1.5919 + MarkRefsIntoAndScanClosure
1.5920 + mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */,
1.5921 + &_markStack, this,
1.5922 + false /* should_yield */, false /* not precleaning */);
1.5923 + MarkFromDirtyCardsClosure
1.5924 + markFromDirtyCardsClosure(this, _span,
1.5925 + NULL, // space is set further below
1.5926 + &_markBitMap, &_markStack, &mrias_cl);
1.5927 + {
1.5928 + GCTraceTime t("grey object rescan", PrintGCDetails, false, _gc_timer_cm);
1.5929 + // Iterate over the dirty cards, setting the corresponding bits in the
1.5930 + // mod union table.
1.5931 + {
1.5932 + ModUnionClosure modUnionClosure(&_modUnionTable);
1.5933 + _ct->ct_bs()->dirty_card_iterate(
1.5934 + _cmsGen->used_region(),
1.5935 + &modUnionClosure);
1.5936 + }
1.5937 + // Having transferred these marks into the modUnionTable, we just need
1.5938 + // to rescan the marked objects on the dirty cards in the modUnionTable.
1.5939 + // The initial marking may have been done during an asynchronous
1.5940 + // collection so there may be dirty bits in the mod-union table.
1.5941 + const int alignment =
1.5942 + CardTableModRefBS::card_size * BitsPerWord;
1.5943 + {
1.5944 + // ... First handle dirty cards in CMS gen
1.5945 + markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
1.5946 + MemRegion ur = _cmsGen->used_region();
1.5947 + HeapWord* lb = ur.start();
1.5948 + HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
1.5949 + MemRegion cms_span(lb, ub);
1.5950 + _modUnionTable.dirty_range_iterate_clear(cms_span,
1.5951 + &markFromDirtyCardsClosure);
1.5952 + verify_work_stacks_empty();
1.5953 + if (PrintCMSStatistics != 0) {
1.5954 + gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ",
1.5955 + markFromDirtyCardsClosure.num_dirty_cards());
1.5956 + }
1.5957 + }
1.5958 + }
1.5959 + if (VerifyDuringGC &&
1.5960 + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
1.5961 + HandleMark hm; // Discard invalid handles created during verification
1.5962 + Universe::verify();
1.5963 + }
1.5964 + {
1.5965 + GCTraceTime t("root rescan", PrintGCDetails, false, _gc_timer_cm);
1.5966 +
1.5967 + verify_work_stacks_empty();
1.5968 +
1.5969 + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
1.5970 + GenCollectedHeap::StrongRootsScope srs(gch);
1.5971 + gch->gen_process_strong_roots(_cmsGen->level(),
1.5972 + true, // younger gens as roots
1.5973 + false, // use the local StrongRootsScope
1.5974 + false, // not scavenging
1.5975 + SharedHeap::ScanningOption(roots_scanning_options()),
1.5976 + &mrias_cl,
1.5977 + true, // walk code active on stacks
1.5978 + NULL,
1.5979 + NULL); // The dirty klasses will be handled below
1.5980 +
1.5981 + assert(should_unload_classes()
1.5982 + || (roots_scanning_options() & SharedHeap::SO_CodeCache),
1.5983 + "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
1.5984 + }
1.5985 +
1.5986 + {
1.5987 + GCTraceTime t("visit unhandled CLDs", PrintGCDetails, false, _gc_timer_cm);
1.5988 +
1.5989 + verify_work_stacks_empty();
1.5990 +
1.5991 + // Scan all class loader data objects that might have been introduced
1.5992 + // during concurrent marking.
1.5993 + ResourceMark rm;
1.5994 + GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
1.5995 + for (int i = 0; i < array->length(); i++) {
1.5996 + mrias_cl.do_class_loader_data(array->at(i));
1.5997 + }
1.5998 +
1.5999 + // We don't need to keep track of new CLDs anymore.
1.6000 + ClassLoaderDataGraph::remember_new_clds(false);
1.6001 +
1.6002 + verify_work_stacks_empty();
1.6003 + }
1.6004 +
1.6005 + {
1.6006 + GCTraceTime t("dirty klass scan", PrintGCDetails, false, _gc_timer_cm);
1.6007 +
1.6008 + verify_work_stacks_empty();
1.6009 +
1.6010 + RemarkKlassClosure remark_klass_closure(&mrias_cl);
1.6011 + ClassLoaderDataGraph::classes_do(&remark_klass_closure);
1.6012 +
1.6013 + verify_work_stacks_empty();
1.6014 + }
1.6015 +
1.6016 + // We might have added oops to ClassLoaderData::_handles during the
1.6017 + // concurrent marking phase. These oops point to newly allocated objects
1.6018 + // that are guaranteed to be kept alive. Either by the direct allocation
1.6019 + // code, or when the young collector processes the strong roots. Hence,
1.6020 + // we don't have to revisit the _handles block during the remark phase.
1.6021 +
1.6022 + verify_work_stacks_empty();
1.6023 + // Restore evacuated mark words, if any, used for overflow list links
1.6024 + if (!CMSOverflowEarlyRestoration) {
1.6025 + restore_preserved_marks_if_any();
1.6026 + }
1.6027 + verify_overflow_empty();
1.6028 +}
1.6029 +
1.6030 +////////////////////////////////////////////////////////
1.6031 +// Parallel Reference Processing Task Proxy Class
1.6032 +////////////////////////////////////////////////////////
1.6033 +class CMSRefProcTaskProxy: public AbstractGangTaskWOopQueues {
1.6034 + typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
1.6035 + CMSCollector* _collector;
1.6036 + CMSBitMap* _mark_bit_map;
1.6037 + const MemRegion _span;
1.6038 + ProcessTask& _task;
1.6039 +
1.6040 +public:
1.6041 + CMSRefProcTaskProxy(ProcessTask& task,
1.6042 + CMSCollector* collector,
1.6043 + const MemRegion& span,
1.6044 + CMSBitMap* mark_bit_map,
1.6045 + AbstractWorkGang* workers,
1.6046 + OopTaskQueueSet* task_queues):
1.6047 + // XXX Should superclass AGTWOQ also know about AWG since it knows
1.6048 + // about the task_queues used by the AWG? Then it could initialize
1.6049 + // the terminator() object. See 6984287. The set_for_termination()
1.6050 + // below is a temporary band-aid for the regression in 6984287.
1.6051 + AbstractGangTaskWOopQueues("Process referents by policy in parallel",
1.6052 + task_queues),
1.6053 + _task(task),
1.6054 + _collector(collector), _span(span), _mark_bit_map(mark_bit_map)
1.6055 + {
1.6056 + assert(_collector->_span.equals(_span) && !_span.is_empty(),
1.6057 + "Inconsistency in _span");
1.6058 + set_for_termination(workers->active_workers());
1.6059 + }
1.6060 +
1.6061 + OopTaskQueueSet* task_queues() { return queues(); }
1.6062 +
1.6063 + OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
1.6064 +
1.6065 + void do_work_steal(int i,
1.6066 + CMSParDrainMarkingStackClosure* drain,
1.6067 + CMSParKeepAliveClosure* keep_alive,
1.6068 + int* seed);
1.6069 +
1.6070 + virtual void work(uint worker_id);
1.6071 +};
1.6072 +
1.6073 +void CMSRefProcTaskProxy::work(uint worker_id) {
1.6074 + assert(_collector->_span.equals(_span), "Inconsistency in _span");
1.6075 + CMSParKeepAliveClosure par_keep_alive(_collector, _span,
1.6076 + _mark_bit_map,
1.6077 + work_queue(worker_id));
1.6078 + CMSParDrainMarkingStackClosure par_drain_stack(_collector, _span,
1.6079 + _mark_bit_map,
1.6080 + work_queue(worker_id));
1.6081 + CMSIsAliveClosure is_alive_closure(_span, _mark_bit_map);
1.6082 + _task.work(worker_id, is_alive_closure, par_keep_alive, par_drain_stack);
1.6083 + if (_task.marks_oops_alive()) {
1.6084 + do_work_steal(worker_id, &par_drain_stack, &par_keep_alive,
1.6085 + _collector->hash_seed(worker_id));
1.6086 + }
1.6087 + assert(work_queue(worker_id)->size() == 0, "work_queue should be empty");
1.6088 + assert(_collector->_overflow_list == NULL, "non-empty _overflow_list");
1.6089 +}
1.6090 +
1.6091 +class CMSRefEnqueueTaskProxy: public AbstractGangTask {
1.6092 + typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
1.6093 + EnqueueTask& _task;
1.6094 +
1.6095 +public:
1.6096 + CMSRefEnqueueTaskProxy(EnqueueTask& task)
1.6097 + : AbstractGangTask("Enqueue reference objects in parallel"),
1.6098 + _task(task)
1.6099 + { }
1.6100 +
1.6101 + virtual void work(uint worker_id)
1.6102 + {
1.6103 + _task.work(worker_id);
1.6104 + }
1.6105 +};
1.6106 +
1.6107 +CMSParKeepAliveClosure::CMSParKeepAliveClosure(CMSCollector* collector,
1.6108 + MemRegion span, CMSBitMap* bit_map, OopTaskQueue* work_queue):
1.6109 + _span(span),
1.6110 + _bit_map(bit_map),
1.6111 + _work_queue(work_queue),
1.6112 + _mark_and_push(collector, span, bit_map, work_queue),
1.6113 + _low_water_mark(MIN2((uint)(work_queue->max_elems()/4),
1.6114 + (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads)))
1.6115 +{ }
1.6116 +
1.6117 +// . see if we can share work_queues with ParNew? XXX
1.6118 +void CMSRefProcTaskProxy::do_work_steal(int i,
1.6119 + CMSParDrainMarkingStackClosure* drain,
1.6120 + CMSParKeepAliveClosure* keep_alive,
1.6121 + int* seed) {
1.6122 + OopTaskQueue* work_q = work_queue(i);
1.6123 + NOT_PRODUCT(int num_steals = 0;)
1.6124 + oop obj_to_scan;
1.6125 +
1.6126 + while (true) {
1.6127 + // Completely finish any left over work from (an) earlier round(s)
1.6128 + drain->trim_queue(0);
1.6129 + size_t num_from_overflow_list = MIN2((size_t)(work_q->max_elems() - work_q->size())/4,
1.6130 + (size_t)ParGCDesiredObjsFromOverflowList);
1.6131 + // Now check if there's any work in the overflow list
1.6132 + // Passing ParallelGCThreads as the third parameter, no_of_gc_threads,
1.6133 + // only affects the number of attempts made to get work from the
1.6134 + // overflow list and does not affect the number of workers. Just
1.6135 + // pass ParallelGCThreads so this behavior is unchanged.
1.6136 + if (_collector->par_take_from_overflow_list(num_from_overflow_list,
1.6137 + work_q,
1.6138 + ParallelGCThreads)) {
1.6139 + // Found something in global overflow list;
1.6140 + // not yet ready to go stealing work from others.
1.6141 + // We'd like to assert(work_q->size() != 0, ...)
1.6142 + // because we just took work from the overflow list,
1.6143 + // but of course we can't, since all of that might have
1.6144 + // been already stolen from us.
1.6145 + continue;
1.6146 + }
1.6147 + // Verify that we have no work before we resort to stealing
1.6148 + assert(work_q->size() == 0, "Have work, shouldn't steal");
1.6149 + // Try to steal from other queues that have work
1.6150 + if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
1.6151 + NOT_PRODUCT(num_steals++;)
1.6152 + assert(obj_to_scan->is_oop(), "Oops, not an oop!");
1.6153 + assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
1.6154 + // Do scanning work
1.6155 + obj_to_scan->oop_iterate(keep_alive);
1.6156 + // Loop around, finish this work, and try to steal some more
1.6157 + } else if (terminator()->offer_termination()) {
1.6158 + break; // nirvana from the infinite cycle
1.6159 + }
1.6160 + }
1.6161 + NOT_PRODUCT(
1.6162 + if (PrintCMSStatistics != 0) {
1.6163 + gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
1.6164 + }
1.6165 + )
1.6166 +}
1.6167 +
1.6168 +void CMSRefProcTaskExecutor::execute(ProcessTask& task)
1.6169 +{
1.6170 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.6171 + FlexibleWorkGang* workers = gch->workers();
1.6172 + assert(workers != NULL, "Need parallel worker threads.");
1.6173 + CMSRefProcTaskProxy rp_task(task, &_collector,
1.6174 + _collector.ref_processor()->span(),
1.6175 + _collector.markBitMap(),
1.6176 + workers, _collector.task_queues());
1.6177 + workers->run_task(&rp_task);
1.6178 +}
1.6179 +
1.6180 +void CMSRefProcTaskExecutor::execute(EnqueueTask& task)
1.6181 +{
1.6182 +
1.6183 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.6184 + FlexibleWorkGang* workers = gch->workers();
1.6185 + assert(workers != NULL, "Need parallel worker threads.");
1.6186 + CMSRefEnqueueTaskProxy enq_task(task);
1.6187 + workers->run_task(&enq_task);
1.6188 +}
1.6189 +
1.6190 +void CMSCollector::refProcessingWork(bool asynch, bool clear_all_soft_refs) {
1.6191 +
1.6192 + ResourceMark rm;
1.6193 + HandleMark hm;
1.6194 +
1.6195 + ReferenceProcessor* rp = ref_processor();
1.6196 + assert(rp->span().equals(_span), "Spans should be equal");
1.6197 + assert(!rp->enqueuing_is_done(), "Enqueuing should not be complete");
1.6198 + // Process weak references.
1.6199 + rp->setup_policy(clear_all_soft_refs);
1.6200 + verify_work_stacks_empty();
1.6201 +
1.6202 + CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap,
1.6203 + &_markStack, false /* !preclean */);
1.6204 + CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this,
1.6205 + _span, &_markBitMap, &_markStack,
1.6206 + &cmsKeepAliveClosure, false /* !preclean */);
1.6207 + {
1.6208 + GCTraceTime t("weak refs processing", PrintGCDetails, false, _gc_timer_cm);
1.6209 +
1.6210 + ReferenceProcessorStats stats;
1.6211 + if (rp->processing_is_mt()) {
1.6212 + // Set the degree of MT here. If the discovery is done MT, there
1.6213 + // may have been a different number of threads doing the discovery
1.6214 + // and a different number of discovered lists may have Ref objects.
1.6215 + // That is OK as long as the Reference lists are balanced (see
1.6216 + // balance_all_queues() and balance_queues()).
1.6217 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.6218 + int active_workers = ParallelGCThreads;
1.6219 + FlexibleWorkGang* workers = gch->workers();
1.6220 + if (workers != NULL) {
1.6221 + active_workers = workers->active_workers();
1.6222 + // The expectation is that active_workers will have already
1.6223 + // been set to a reasonable value. If it has not been set,
1.6224 + // investigate.
1.6225 + assert(active_workers > 0, "Should have been set during scavenge");
1.6226 + }
1.6227 + rp->set_active_mt_degree(active_workers);
1.6228 + CMSRefProcTaskExecutor task_executor(*this);
1.6229 + stats = rp->process_discovered_references(&_is_alive_closure,
1.6230 + &cmsKeepAliveClosure,
1.6231 + &cmsDrainMarkingStackClosure,
1.6232 + &task_executor,
1.6233 + _gc_timer_cm);
1.6234 + } else {
1.6235 + stats = rp->process_discovered_references(&_is_alive_closure,
1.6236 + &cmsKeepAliveClosure,
1.6237 + &cmsDrainMarkingStackClosure,
1.6238 + NULL,
1.6239 + _gc_timer_cm);
1.6240 + }
1.6241 + _gc_tracer_cm->report_gc_reference_stats(stats);
1.6242 +
1.6243 + }
1.6244 +
1.6245 + // This is the point where the entire marking should have completed.
1.6246 + verify_work_stacks_empty();
1.6247 +
1.6248 + if (should_unload_classes()) {
1.6249 + {
1.6250 + GCTraceTime t("class unloading", PrintGCDetails, false, _gc_timer_cm);
1.6251 +
1.6252 + // Unload classes and purge the SystemDictionary.
1.6253 + bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure);
1.6254 +
1.6255 + // Unload nmethods.
1.6256 + CodeCache::do_unloading(&_is_alive_closure, purged_class);
1.6257 +
1.6258 + // Prune dead klasses from subklass/sibling/implementor lists.
1.6259 + Klass::clean_weak_klass_links(&_is_alive_closure);
1.6260 + }
1.6261 +
1.6262 + {
1.6263 + GCTraceTime t("scrub symbol table", PrintGCDetails, false, _gc_timer_cm);
1.6264 + // Clean up unreferenced symbols in symbol table.
1.6265 + SymbolTable::unlink();
1.6266 + }
1.6267 + }
1.6268 +
1.6269 + // CMS doesn't use the StringTable as hard roots when class unloading is turned off.
1.6270 + // Need to check if we really scanned the StringTable.
1.6271 + if ((roots_scanning_options() & SharedHeap::SO_Strings) == 0) {
1.6272 + GCTraceTime t("scrub string table", PrintGCDetails, false, _gc_timer_cm);
1.6273 + // Delete entries for dead interned strings.
1.6274 + StringTable::unlink(&_is_alive_closure);
1.6275 + }
1.6276 +
1.6277 + // Restore any preserved marks as a result of mark stack or
1.6278 + // work queue overflow
1.6279 + restore_preserved_marks_if_any(); // done single-threaded for now
1.6280 +
1.6281 + rp->set_enqueuing_is_done(true);
1.6282 + if (rp->processing_is_mt()) {
1.6283 + rp->balance_all_queues();
1.6284 + CMSRefProcTaskExecutor task_executor(*this);
1.6285 + rp->enqueue_discovered_references(&task_executor);
1.6286 + } else {
1.6287 + rp->enqueue_discovered_references(NULL);
1.6288 + }
1.6289 + rp->verify_no_references_recorded();
1.6290 + assert(!rp->discovery_enabled(), "should have been disabled");
1.6291 +}
1.6292 +
1.6293 +#ifndef PRODUCT
1.6294 +void CMSCollector::check_correct_thread_executing() {
1.6295 + Thread* t = Thread::current();
1.6296 + // Only the VM thread or the CMS thread should be here.
1.6297 + assert(t->is_ConcurrentGC_thread() || t->is_VM_thread(),
1.6298 + "Unexpected thread type");
1.6299 + // If this is the vm thread, the foreground process
1.6300 + // should not be waiting. Note that _foregroundGCIsActive is
1.6301 + // true while the foreground collector is waiting.
1.6302 + if (_foregroundGCShouldWait) {
1.6303 + // We cannot be the VM thread
1.6304 + assert(t->is_ConcurrentGC_thread(),
1.6305 + "Should be CMS thread");
1.6306 + } else {
1.6307 + // We can be the CMS thread only if we are in a stop-world
1.6308 + // phase of CMS collection.
1.6309 + if (t->is_ConcurrentGC_thread()) {
1.6310 + assert(_collectorState == InitialMarking ||
1.6311 + _collectorState == FinalMarking,
1.6312 + "Should be a stop-world phase");
1.6313 + // The CMS thread should be holding the CMS_token.
1.6314 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.6315 + "Potential interference with concurrently "
1.6316 + "executing VM thread");
1.6317 + }
1.6318 + }
1.6319 +}
1.6320 +#endif
1.6321 +
1.6322 +void CMSCollector::sweep(bool asynch) {
1.6323 + assert(_collectorState == Sweeping, "just checking");
1.6324 + check_correct_thread_executing();
1.6325 + verify_work_stacks_empty();
1.6326 + verify_overflow_empty();
1.6327 + increment_sweep_count();
1.6328 + TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
1.6329 +
1.6330 + _inter_sweep_timer.stop();
1.6331 + _inter_sweep_estimate.sample(_inter_sweep_timer.seconds());
1.6332 + size_policy()->avg_cms_free_at_sweep()->sample(_cmsGen->free());
1.6333 +
1.6334 + assert(!_intra_sweep_timer.is_active(), "Should not be active");
1.6335 + _intra_sweep_timer.reset();
1.6336 + _intra_sweep_timer.start();
1.6337 + if (asynch) {
1.6338 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.6339 + CMSPhaseAccounting pa(this, "sweep", !PrintGCDetails);
1.6340 + // First sweep the old gen
1.6341 + {
1.6342 + CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
1.6343 + bitMapLock());
1.6344 + sweepWork(_cmsGen, asynch);
1.6345 + }
1.6346 +
1.6347 + // Update Universe::_heap_*_at_gc figures.
1.6348 + // We need all the free list locks to make the abstract state
1.6349 + // transition from Sweeping to Resetting. See detailed note
1.6350 + // further below.
1.6351 + {
1.6352 + CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock());
1.6353 + // Update heap occupancy information which is used as
1.6354 + // input to soft ref clearing policy at the next gc.
1.6355 + Universe::update_heap_info_at_gc();
1.6356 + _collectorState = Resizing;
1.6357 + }
1.6358 + } else {
1.6359 + // already have needed locks
1.6360 + sweepWork(_cmsGen, asynch);
1.6361 + // Update heap occupancy information which is used as
1.6362 + // input to soft ref clearing policy at the next gc.
1.6363 + Universe::update_heap_info_at_gc();
1.6364 + _collectorState = Resizing;
1.6365 + }
1.6366 + verify_work_stacks_empty();
1.6367 + verify_overflow_empty();
1.6368 +
1.6369 + if (should_unload_classes()) {
1.6370 + // Delay purge to the beginning of the next safepoint. Metaspace::contains
1.6371 + // requires that the virtual spaces are stable and not deleted.
1.6372 + ClassLoaderDataGraph::set_should_purge(true);
1.6373 + }
1.6374 +
1.6375 + _intra_sweep_timer.stop();
1.6376 + _intra_sweep_estimate.sample(_intra_sweep_timer.seconds());
1.6377 +
1.6378 + _inter_sweep_timer.reset();
1.6379 + _inter_sweep_timer.start();
1.6380 +
1.6381 + // We need to use a monotonically non-deccreasing time in ms
1.6382 + // or we will see time-warp warnings and os::javaTimeMillis()
1.6383 + // does not guarantee monotonicity.
1.6384 + jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC;
1.6385 + update_time_of_last_gc(now);
1.6386 +
1.6387 + // NOTE on abstract state transitions:
1.6388 + // Mutators allocate-live and/or mark the mod-union table dirty
1.6389 + // based on the state of the collection. The former is done in
1.6390 + // the interval [Marking, Sweeping] and the latter in the interval
1.6391 + // [Marking, Sweeping). Thus the transitions into the Marking state
1.6392 + // and out of the Sweeping state must be synchronously visible
1.6393 + // globally to the mutators.
1.6394 + // The transition into the Marking state happens with the world
1.6395 + // stopped so the mutators will globally see it. Sweeping is
1.6396 + // done asynchronously by the background collector so the transition
1.6397 + // from the Sweeping state to the Resizing state must be done
1.6398 + // under the freelistLock (as is the check for whether to
1.6399 + // allocate-live and whether to dirty the mod-union table).
1.6400 + assert(_collectorState == Resizing, "Change of collector state to"
1.6401 + " Resizing must be done under the freelistLocks (plural)");
1.6402 +
1.6403 + // Now that sweeping has been completed, we clear
1.6404 + // the incremental_collection_failed flag,
1.6405 + // thus inviting a younger gen collection to promote into
1.6406 + // this generation. If such a promotion may still fail,
1.6407 + // the flag will be set again when a young collection is
1.6408 + // attempted.
1.6409 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.6410 + gch->clear_incremental_collection_failed(); // Worth retrying as fresh space may have been freed up
1.6411 + gch->update_full_collections_completed(_collection_count_start);
1.6412 +}
1.6413 +
1.6414 +// FIX ME!!! Looks like this belongs in CFLSpace, with
1.6415 +// CMSGen merely delegating to it.
1.6416 +void ConcurrentMarkSweepGeneration::setNearLargestChunk() {
1.6417 + double nearLargestPercent = FLSLargestBlockCoalesceProximity;
1.6418 + HeapWord* minAddr = _cmsSpace->bottom();
1.6419 + HeapWord* largestAddr =
1.6420 + (HeapWord*) _cmsSpace->dictionary()->find_largest_dict();
1.6421 + if (largestAddr == NULL) {
1.6422 + // The dictionary appears to be empty. In this case
1.6423 + // try to coalesce at the end of the heap.
1.6424 + largestAddr = _cmsSpace->end();
1.6425 + }
1.6426 + size_t largestOffset = pointer_delta(largestAddr, minAddr);
1.6427 + size_t nearLargestOffset =
1.6428 + (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
1.6429 + if (PrintFLSStatistics != 0) {
1.6430 + gclog_or_tty->print_cr(
1.6431 + "CMS: Large Block: " PTR_FORMAT ";"
1.6432 + " Proximity: " PTR_FORMAT " -> " PTR_FORMAT,
1.6433 + largestAddr,
1.6434 + _cmsSpace->nearLargestChunk(), minAddr + nearLargestOffset);
1.6435 + }
1.6436 + _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
1.6437 +}
1.6438 +
1.6439 +bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
1.6440 + return addr >= _cmsSpace->nearLargestChunk();
1.6441 +}
1.6442 +
1.6443 +FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
1.6444 + return _cmsSpace->find_chunk_at_end();
1.6445 +}
1.6446 +
1.6447 +void ConcurrentMarkSweepGeneration::update_gc_stats(int current_level,
1.6448 + bool full) {
1.6449 + // The next lower level has been collected. Gather any statistics
1.6450 + // that are of interest at this point.
1.6451 + if (!full && (current_level + 1) == level()) {
1.6452 + // Gather statistics on the young generation collection.
1.6453 + collector()->stats().record_gc0_end(used());
1.6454 + }
1.6455 +}
1.6456 +
1.6457 +CMSAdaptiveSizePolicy* ConcurrentMarkSweepGeneration::size_policy() {
1.6458 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.6459 + assert(gch->kind() == CollectedHeap::GenCollectedHeap,
1.6460 + "Wrong type of heap");
1.6461 + CMSAdaptiveSizePolicy* sp = (CMSAdaptiveSizePolicy*)
1.6462 + gch->gen_policy()->size_policy();
1.6463 + assert(sp->is_gc_cms_adaptive_size_policy(),
1.6464 + "Wrong type of size policy");
1.6465 + return sp;
1.6466 +}
1.6467 +
1.6468 +void ConcurrentMarkSweepGeneration::rotate_debug_collection_type() {
1.6469 + if (PrintGCDetails && Verbose) {
1.6470 + gclog_or_tty->print("Rotate from %d ", _debug_collection_type);
1.6471 + }
1.6472 + _debug_collection_type = (CollectionTypes) (_debug_collection_type + 1);
1.6473 + _debug_collection_type =
1.6474 + (CollectionTypes) (_debug_collection_type % Unknown_collection_type);
1.6475 + if (PrintGCDetails && Verbose) {
1.6476 + gclog_or_tty->print_cr("to %d ", _debug_collection_type);
1.6477 + }
1.6478 +}
1.6479 +
1.6480 +void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* gen,
1.6481 + bool asynch) {
1.6482 + // We iterate over the space(s) underlying this generation,
1.6483 + // checking the mark bit map to see if the bits corresponding
1.6484 + // to specific blocks are marked or not. Blocks that are
1.6485 + // marked are live and are not swept up. All remaining blocks
1.6486 + // are swept up, with coalescing on-the-fly as we sweep up
1.6487 + // contiguous free and/or garbage blocks:
1.6488 + // We need to ensure that the sweeper synchronizes with allocators
1.6489 + // and stop-the-world collectors. In particular, the following
1.6490 + // locks are used:
1.6491 + // . CMS token: if this is held, a stop the world collection cannot occur
1.6492 + // . freelistLock: if this is held no allocation can occur from this
1.6493 + // generation by another thread
1.6494 + // . bitMapLock: if this is held, no other thread can access or update
1.6495 + //
1.6496 +
1.6497 + // Note that we need to hold the freelistLock if we use
1.6498 + // block iterate below; else the iterator might go awry if
1.6499 + // a mutator (or promotion) causes block contents to change
1.6500 + // (for instance if the allocator divvies up a block).
1.6501 + // If we hold the free list lock, for all practical purposes
1.6502 + // young generation GC's can't occur (they'll usually need to
1.6503 + // promote), so we might as well prevent all young generation
1.6504 + // GC's while we do a sweeping step. For the same reason, we might
1.6505 + // as well take the bit map lock for the entire duration
1.6506 +
1.6507 + // check that we hold the requisite locks
1.6508 + assert(have_cms_token(), "Should hold cms token");
1.6509 + assert( (asynch && ConcurrentMarkSweepThread::cms_thread_has_cms_token())
1.6510 + || (!asynch && ConcurrentMarkSweepThread::vm_thread_has_cms_token()),
1.6511 + "Should possess CMS token to sweep");
1.6512 + assert_lock_strong(gen->freelistLock());
1.6513 + assert_lock_strong(bitMapLock());
1.6514 +
1.6515 + assert(!_inter_sweep_timer.is_active(), "Was switched off in an outer context");
1.6516 + assert(_intra_sweep_timer.is_active(), "Was switched on in an outer context");
1.6517 + gen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()),
1.6518 + _inter_sweep_estimate.padded_average(),
1.6519 + _intra_sweep_estimate.padded_average());
1.6520 + gen->setNearLargestChunk();
1.6521 +
1.6522 + {
1.6523 + SweepClosure sweepClosure(this, gen, &_markBitMap,
1.6524 + CMSYield && asynch);
1.6525 + gen->cmsSpace()->blk_iterate_careful(&sweepClosure);
1.6526 + // We need to free-up/coalesce garbage/blocks from a
1.6527 + // co-terminal free run. This is done in the SweepClosure
1.6528 + // destructor; so, do not remove this scope, else the
1.6529 + // end-of-sweep-census below will be off by a little bit.
1.6530 + }
1.6531 + gen->cmsSpace()->sweep_completed();
1.6532 + gen->cmsSpace()->endSweepFLCensus(sweep_count());
1.6533 + if (should_unload_classes()) { // unloaded classes this cycle,
1.6534 + _concurrent_cycles_since_last_unload = 0; // ... reset count
1.6535 + } else { // did not unload classes,
1.6536 + _concurrent_cycles_since_last_unload++; // ... increment count
1.6537 + }
1.6538 +}
1.6539 +
1.6540 +// Reset CMS data structures (for now just the marking bit map)
1.6541 +// preparatory for the next cycle.
1.6542 +void CMSCollector::reset(bool asynch) {
1.6543 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.6544 + CMSAdaptiveSizePolicy* sp = size_policy();
1.6545 + AdaptiveSizePolicyOutput(sp, gch->total_collections());
1.6546 + if (asynch) {
1.6547 + CMSTokenSyncWithLocks ts(true, bitMapLock());
1.6548 +
1.6549 + // If the state is not "Resetting", the foreground thread
1.6550 + // has done a collection and the resetting.
1.6551 + if (_collectorState != Resetting) {
1.6552 + assert(_collectorState == Idling, "The state should only change"
1.6553 + " because the foreground collector has finished the collection");
1.6554 + return;
1.6555 + }
1.6556 +
1.6557 + // Clear the mark bitmap (no grey objects to start with)
1.6558 + // for the next cycle.
1.6559 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.6560 + CMSPhaseAccounting cmspa(this, "reset", !PrintGCDetails);
1.6561 +
1.6562 + HeapWord* curAddr = _markBitMap.startWord();
1.6563 + while (curAddr < _markBitMap.endWord()) {
1.6564 + size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr);
1.6565 + MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining));
1.6566 + _markBitMap.clear_large_range(chunk);
1.6567 + if (ConcurrentMarkSweepThread::should_yield() &&
1.6568 + !foregroundGCIsActive() &&
1.6569 + CMSYield) {
1.6570 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.6571 + "CMS thread should hold CMS token");
1.6572 + assert_lock_strong(bitMapLock());
1.6573 + bitMapLock()->unlock();
1.6574 + ConcurrentMarkSweepThread::desynchronize(true);
1.6575 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.6576 + stopTimer();
1.6577 + if (PrintCMSStatistics != 0) {
1.6578 + incrementYields();
1.6579 + }
1.6580 + icms_wait();
1.6581 +
1.6582 + // See the comment in coordinator_yield()
1.6583 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.6584 + ConcurrentMarkSweepThread::should_yield() &&
1.6585 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.6586 + os::sleep(Thread::current(), 1, false);
1.6587 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.6588 + }
1.6589 +
1.6590 + ConcurrentMarkSweepThread::synchronize(true);
1.6591 + bitMapLock()->lock_without_safepoint_check();
1.6592 + startTimer();
1.6593 + }
1.6594 + curAddr = chunk.end();
1.6595 + }
1.6596 + // A successful mostly concurrent collection has been done.
1.6597 + // Because only the full (i.e., concurrent mode failure) collections
1.6598 + // are being measured for gc overhead limits, clean the "near" flag
1.6599 + // and count.
1.6600 + sp->reset_gc_overhead_limit_count();
1.6601 + _collectorState = Idling;
1.6602 + } else {
1.6603 + // already have the lock
1.6604 + assert(_collectorState == Resetting, "just checking");
1.6605 + assert_lock_strong(bitMapLock());
1.6606 + _markBitMap.clear_all();
1.6607 + _collectorState = Idling;
1.6608 + }
1.6609 +
1.6610 + // Stop incremental mode after a cycle completes, so that any future cycles
1.6611 + // are triggered by allocation.
1.6612 + stop_icms();
1.6613 +
1.6614 + NOT_PRODUCT(
1.6615 + if (RotateCMSCollectionTypes) {
1.6616 + _cmsGen->rotate_debug_collection_type();
1.6617 + }
1.6618 + )
1.6619 +
1.6620 + register_gc_end();
1.6621 +}
1.6622 +
1.6623 +void CMSCollector::do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause) {
1.6624 + gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
1.6625 + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
1.6626 + GCTraceTime t(GCCauseString("GC", gc_cause), PrintGC, !PrintGCDetails, NULL);
1.6627 + TraceCollectorStats tcs(counters());
1.6628 +
1.6629 + switch (op) {
1.6630 + case CMS_op_checkpointRootsInitial: {
1.6631 + SvcGCMarker sgcm(SvcGCMarker::OTHER);
1.6632 + checkpointRootsInitial(true); // asynch
1.6633 + if (PrintGC) {
1.6634 + _cmsGen->printOccupancy("initial-mark");
1.6635 + }
1.6636 + break;
1.6637 + }
1.6638 + case CMS_op_checkpointRootsFinal: {
1.6639 + SvcGCMarker sgcm(SvcGCMarker::OTHER);
1.6640 + checkpointRootsFinal(true, // asynch
1.6641 + false, // !clear_all_soft_refs
1.6642 + false); // !init_mark_was_synchronous
1.6643 + if (PrintGC) {
1.6644 + _cmsGen->printOccupancy("remark");
1.6645 + }
1.6646 + break;
1.6647 + }
1.6648 + default:
1.6649 + fatal("No such CMS_op");
1.6650 + }
1.6651 +}
1.6652 +
1.6653 +#ifndef PRODUCT
1.6654 +size_t const CMSCollector::skip_header_HeapWords() {
1.6655 + return FreeChunk::header_size();
1.6656 +}
1.6657 +
1.6658 +// Try and collect here conditions that should hold when
1.6659 +// CMS thread is exiting. The idea is that the foreground GC
1.6660 +// thread should not be blocked if it wants to terminate
1.6661 +// the CMS thread and yet continue to run the VM for a while
1.6662 +// after that.
1.6663 +void CMSCollector::verify_ok_to_terminate() const {
1.6664 + assert(Thread::current()->is_ConcurrentGC_thread(),
1.6665 + "should be called by CMS thread");
1.6666 + assert(!_foregroundGCShouldWait, "should be false");
1.6667 + // We could check here that all the various low-level locks
1.6668 + // are not held by the CMS thread, but that is overkill; see
1.6669 + // also CMSThread::verify_ok_to_terminate() where the CGC_lock
1.6670 + // is checked.
1.6671 +}
1.6672 +#endif
1.6673 +
1.6674 +size_t CMSCollector::block_size_using_printezis_bits(HeapWord* addr) const {
1.6675 + assert(_markBitMap.isMarked(addr) && _markBitMap.isMarked(addr + 1),
1.6676 + "missing Printezis mark?");
1.6677 + HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
1.6678 + size_t size = pointer_delta(nextOneAddr + 1, addr);
1.6679 + assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
1.6680 + "alignment problem");
1.6681 + assert(size >= 3, "Necessary for Printezis marks to work");
1.6682 + return size;
1.6683 +}
1.6684 +
1.6685 +// A variant of the above (block_size_using_printezis_bits()) except
1.6686 +// that we return 0 if the P-bits are not yet set.
1.6687 +size_t CMSCollector::block_size_if_printezis_bits(HeapWord* addr) const {
1.6688 + if (_markBitMap.isMarked(addr + 1)) {
1.6689 + assert(_markBitMap.isMarked(addr), "P-bit can be set only for marked objects");
1.6690 + HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
1.6691 + size_t size = pointer_delta(nextOneAddr + 1, addr);
1.6692 + assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
1.6693 + "alignment problem");
1.6694 + assert(size >= 3, "Necessary for Printezis marks to work");
1.6695 + return size;
1.6696 + }
1.6697 + return 0;
1.6698 +}
1.6699 +
1.6700 +HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
1.6701 + size_t sz = 0;
1.6702 + oop p = (oop)addr;
1.6703 + if (p->klass_or_null() != NULL) {
1.6704 + sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
1.6705 + } else {
1.6706 + sz = block_size_using_printezis_bits(addr);
1.6707 + }
1.6708 + assert(sz > 0, "size must be nonzero");
1.6709 + HeapWord* next_block = addr + sz;
1.6710 + HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block,
1.6711 + CardTableModRefBS::card_size);
1.6712 + assert(round_down((uintptr_t)addr, CardTableModRefBS::card_size) <
1.6713 + round_down((uintptr_t)next_card, CardTableModRefBS::card_size),
1.6714 + "must be different cards");
1.6715 + return next_card;
1.6716 +}
1.6717 +
1.6718 +
1.6719 +// CMS Bit Map Wrapper /////////////////////////////////////////
1.6720 +
1.6721 +// Construct a CMS bit map infrastructure, but don't create the
1.6722 +// bit vector itself. That is done by a separate call CMSBitMap::allocate()
1.6723 +// further below.
1.6724 +CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
1.6725 + _bm(),
1.6726 + _shifter(shifter),
1.6727 + _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true) : NULL)
1.6728 +{
1.6729 + _bmStartWord = 0;
1.6730 + _bmWordSize = 0;
1.6731 +}
1.6732 +
1.6733 +bool CMSBitMap::allocate(MemRegion mr) {
1.6734 + _bmStartWord = mr.start();
1.6735 + _bmWordSize = mr.word_size();
1.6736 + ReservedSpace brs(ReservedSpace::allocation_align_size_up(
1.6737 + (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1));
1.6738 + if (!brs.is_reserved()) {
1.6739 + warning("CMS bit map allocation failure");
1.6740 + return false;
1.6741 + }
1.6742 + // For now we'll just commit all of the bit map up fromt.
1.6743 + // Later on we'll try to be more parsimonious with swap.
1.6744 + if (!_virtual_space.initialize(brs, brs.size())) {
1.6745 + warning("CMS bit map backing store failure");
1.6746 + return false;
1.6747 + }
1.6748 + assert(_virtual_space.committed_size() == brs.size(),
1.6749 + "didn't reserve backing store for all of CMS bit map?");
1.6750 + _bm.set_map((BitMap::bm_word_t*)_virtual_space.low());
1.6751 + assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
1.6752 + _bmWordSize, "inconsistency in bit map sizing");
1.6753 + _bm.set_size(_bmWordSize >> _shifter);
1.6754 +
1.6755 + // bm.clear(); // can we rely on getting zero'd memory? verify below
1.6756 + assert(isAllClear(),
1.6757 + "Expected zero'd memory from ReservedSpace constructor");
1.6758 + assert(_bm.size() == heapWordDiffToOffsetDiff(sizeInWords()),
1.6759 + "consistency check");
1.6760 + return true;
1.6761 +}
1.6762 +
1.6763 +void CMSBitMap::dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl) {
1.6764 + HeapWord *next_addr, *end_addr, *last_addr;
1.6765 + assert_locked();
1.6766 + assert(covers(mr), "out-of-range error");
1.6767 + // XXX assert that start and end are appropriately aligned
1.6768 + for (next_addr = mr.start(), end_addr = mr.end();
1.6769 + next_addr < end_addr; next_addr = last_addr) {
1.6770 + MemRegion dirty_region = getAndClearMarkedRegion(next_addr, end_addr);
1.6771 + last_addr = dirty_region.end();
1.6772 + if (!dirty_region.is_empty()) {
1.6773 + cl->do_MemRegion(dirty_region);
1.6774 + } else {
1.6775 + assert(last_addr == end_addr, "program logic");
1.6776 + return;
1.6777 + }
1.6778 + }
1.6779 +}
1.6780 +
1.6781 +void CMSBitMap::print_on_error(outputStream* st, const char* prefix) const {
1.6782 + _bm.print_on_error(st, prefix);
1.6783 +}
1.6784 +
1.6785 +#ifndef PRODUCT
1.6786 +void CMSBitMap::assert_locked() const {
1.6787 + CMSLockVerifier::assert_locked(lock());
1.6788 +}
1.6789 +
1.6790 +bool CMSBitMap::covers(MemRegion mr) const {
1.6791 + // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
1.6792 + assert((size_t)_bm.size() == (_bmWordSize >> _shifter),
1.6793 + "size inconsistency");
1.6794 + return (mr.start() >= _bmStartWord) &&
1.6795 + (mr.end() <= endWord());
1.6796 +}
1.6797 +
1.6798 +bool CMSBitMap::covers(HeapWord* start, size_t size) const {
1.6799 + return (start >= _bmStartWord && (start + size) <= endWord());
1.6800 +}
1.6801 +
1.6802 +void CMSBitMap::verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) {
1.6803 + // verify that there are no 1 bits in the interval [left, right)
1.6804 + FalseBitMapClosure falseBitMapClosure;
1.6805 + iterate(&falseBitMapClosure, left, right);
1.6806 +}
1.6807 +
1.6808 +void CMSBitMap::region_invariant(MemRegion mr)
1.6809 +{
1.6810 + assert_locked();
1.6811 + // mr = mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
1.6812 + assert(!mr.is_empty(), "unexpected empty region");
1.6813 + assert(covers(mr), "mr should be covered by bit map");
1.6814 + // convert address range into offset range
1.6815 + size_t start_ofs = heapWordToOffset(mr.start());
1.6816 + // Make sure that end() is appropriately aligned
1.6817 + assert(mr.end() == (HeapWord*)round_to((intptr_t)mr.end(),
1.6818 + (1 << (_shifter+LogHeapWordSize))),
1.6819 + "Misaligned mr.end()");
1.6820 + size_t end_ofs = heapWordToOffset(mr.end());
1.6821 + assert(end_ofs > start_ofs, "Should mark at least one bit");
1.6822 +}
1.6823 +
1.6824 +#endif
1.6825 +
1.6826 +bool CMSMarkStack::allocate(size_t size) {
1.6827 + // allocate a stack of the requisite depth
1.6828 + ReservedSpace rs(ReservedSpace::allocation_align_size_up(
1.6829 + size * sizeof(oop)));
1.6830 + if (!rs.is_reserved()) {
1.6831 + warning("CMSMarkStack allocation failure");
1.6832 + return false;
1.6833 + }
1.6834 + if (!_virtual_space.initialize(rs, rs.size())) {
1.6835 + warning("CMSMarkStack backing store failure");
1.6836 + return false;
1.6837 + }
1.6838 + assert(_virtual_space.committed_size() == rs.size(),
1.6839 + "didn't reserve backing store for all of CMS stack?");
1.6840 + _base = (oop*)(_virtual_space.low());
1.6841 + _index = 0;
1.6842 + _capacity = size;
1.6843 + NOT_PRODUCT(_max_depth = 0);
1.6844 + return true;
1.6845 +}
1.6846 +
1.6847 +// XXX FIX ME !!! In the MT case we come in here holding a
1.6848 +// leaf lock. For printing we need to take a further lock
1.6849 +// which has lower rank. We need to recallibrate the two
1.6850 +// lock-ranks involved in order to be able to rpint the
1.6851 +// messages below. (Or defer the printing to the caller.
1.6852 +// For now we take the expedient path of just disabling the
1.6853 +// messages for the problematic case.)
1.6854 +void CMSMarkStack::expand() {
1.6855 + assert(_capacity <= MarkStackSizeMax, "stack bigger than permitted");
1.6856 + if (_capacity == MarkStackSizeMax) {
1.6857 + if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
1.6858 + // We print a warning message only once per CMS cycle.
1.6859 + gclog_or_tty->print_cr(" (benign) Hit CMSMarkStack max size limit");
1.6860 + }
1.6861 + return;
1.6862 + }
1.6863 + // Double capacity if possible
1.6864 + size_t new_capacity = MIN2(_capacity*2, MarkStackSizeMax);
1.6865 + // Do not give up existing stack until we have managed to
1.6866 + // get the double capacity that we desired.
1.6867 + ReservedSpace rs(ReservedSpace::allocation_align_size_up(
1.6868 + new_capacity * sizeof(oop)));
1.6869 + if (rs.is_reserved()) {
1.6870 + // Release the backing store associated with old stack
1.6871 + _virtual_space.release();
1.6872 + // Reinitialize virtual space for new stack
1.6873 + if (!_virtual_space.initialize(rs, rs.size())) {
1.6874 + fatal("Not enough swap for expanded marking stack");
1.6875 + }
1.6876 + _base = (oop*)(_virtual_space.low());
1.6877 + _index = 0;
1.6878 + _capacity = new_capacity;
1.6879 + } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
1.6880 + // Failed to double capacity, continue;
1.6881 + // we print a detail message only once per CMS cycle.
1.6882 + gclog_or_tty->print(" (benign) Failed to expand marking stack from "SIZE_FORMAT"K to "
1.6883 + SIZE_FORMAT"K",
1.6884 + _capacity / K, new_capacity / K);
1.6885 + }
1.6886 +}
1.6887 +
1.6888 +
1.6889 +// Closures
1.6890 +// XXX: there seems to be a lot of code duplication here;
1.6891 +// should refactor and consolidate common code.
1.6892 +
1.6893 +// This closure is used to mark refs into the CMS generation in
1.6894 +// the CMS bit map. Called at the first checkpoint. This closure
1.6895 +// assumes that we do not need to re-mark dirty cards; if the CMS
1.6896 +// generation on which this is used is not an oldest
1.6897 +// generation then this will lose younger_gen cards!
1.6898 +
1.6899 +MarkRefsIntoClosure::MarkRefsIntoClosure(
1.6900 + MemRegion span, CMSBitMap* bitMap):
1.6901 + _span(span),
1.6902 + _bitMap(bitMap)
1.6903 +{
1.6904 + assert(_ref_processor == NULL, "deliberately left NULL");
1.6905 + assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
1.6906 +}
1.6907 +
1.6908 +void MarkRefsIntoClosure::do_oop(oop obj) {
1.6909 + // if p points into _span, then mark corresponding bit in _markBitMap
1.6910 + assert(obj->is_oop(), "expected an oop");
1.6911 + HeapWord* addr = (HeapWord*)obj;
1.6912 + if (_span.contains(addr)) {
1.6913 + // this should be made more efficient
1.6914 + _bitMap->mark(addr);
1.6915 + }
1.6916 +}
1.6917 +
1.6918 +void MarkRefsIntoClosure::do_oop(oop* p) { MarkRefsIntoClosure::do_oop_work(p); }
1.6919 +void MarkRefsIntoClosure::do_oop(narrowOop* p) { MarkRefsIntoClosure::do_oop_work(p); }
1.6920 +
1.6921 +Par_MarkRefsIntoClosure::Par_MarkRefsIntoClosure(
1.6922 + MemRegion span, CMSBitMap* bitMap):
1.6923 + _span(span),
1.6924 + _bitMap(bitMap)
1.6925 +{
1.6926 + assert(_ref_processor == NULL, "deliberately left NULL");
1.6927 + assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
1.6928 +}
1.6929 +
1.6930 +void Par_MarkRefsIntoClosure::do_oop(oop obj) {
1.6931 + // if p points into _span, then mark corresponding bit in _markBitMap
1.6932 + assert(obj->is_oop(), "expected an oop");
1.6933 + HeapWord* addr = (HeapWord*)obj;
1.6934 + if (_span.contains(addr)) {
1.6935 + // this should be made more efficient
1.6936 + _bitMap->par_mark(addr);
1.6937 + }
1.6938 +}
1.6939 +
1.6940 +void Par_MarkRefsIntoClosure::do_oop(oop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); }
1.6941 +void Par_MarkRefsIntoClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); }
1.6942 +
1.6943 +// A variant of the above, used for CMS marking verification.
1.6944 +MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
1.6945 + MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
1.6946 + _span(span),
1.6947 + _verification_bm(verification_bm),
1.6948 + _cms_bm(cms_bm)
1.6949 +{
1.6950 + assert(_ref_processor == NULL, "deliberately left NULL");
1.6951 + assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
1.6952 +}
1.6953 +
1.6954 +void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
1.6955 + // if p points into _span, then mark corresponding bit in _markBitMap
1.6956 + assert(obj->is_oop(), "expected an oop");
1.6957 + HeapWord* addr = (HeapWord*)obj;
1.6958 + if (_span.contains(addr)) {
1.6959 + _verification_bm->mark(addr);
1.6960 + if (!_cms_bm->isMarked(addr)) {
1.6961 + oop(addr)->print();
1.6962 + gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", addr);
1.6963 + fatal("... aborting");
1.6964 + }
1.6965 + }
1.6966 +}
1.6967 +
1.6968 +void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
1.6969 +void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
1.6970 +
1.6971 +//////////////////////////////////////////////////
1.6972 +// MarkRefsIntoAndScanClosure
1.6973 +//////////////////////////////////////////////////
1.6974 +
1.6975 +MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
1.6976 + ReferenceProcessor* rp,
1.6977 + CMSBitMap* bit_map,
1.6978 + CMSBitMap* mod_union_table,
1.6979 + CMSMarkStack* mark_stack,
1.6980 + CMSCollector* collector,
1.6981 + bool should_yield,
1.6982 + bool concurrent_precleaning):
1.6983 + _collector(collector),
1.6984 + _span(span),
1.6985 + _bit_map(bit_map),
1.6986 + _mark_stack(mark_stack),
1.6987 + _pushAndMarkClosure(collector, span, rp, bit_map, mod_union_table,
1.6988 + mark_stack, concurrent_precleaning),
1.6989 + _yield(should_yield),
1.6990 + _concurrent_precleaning(concurrent_precleaning),
1.6991 + _freelistLock(NULL)
1.6992 +{
1.6993 + _ref_processor = rp;
1.6994 + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
1.6995 +}
1.6996 +
1.6997 +// This closure is used to mark refs into the CMS generation at the
1.6998 +// second (final) checkpoint, and to scan and transitively follow
1.6999 +// the unmarked oops. It is also used during the concurrent precleaning
1.7000 +// phase while scanning objects on dirty cards in the CMS generation.
1.7001 +// The marks are made in the marking bit map and the marking stack is
1.7002 +// used for keeping the (newly) grey objects during the scan.
1.7003 +// The parallel version (Par_...) appears further below.
1.7004 +void MarkRefsIntoAndScanClosure::do_oop(oop obj) {
1.7005 + if (obj != NULL) {
1.7006 + assert(obj->is_oop(), "expected an oop");
1.7007 + HeapWord* addr = (HeapWord*)obj;
1.7008 + assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)");
1.7009 + assert(_collector->overflow_list_is_empty(),
1.7010 + "overflow list should be empty");
1.7011 + if (_span.contains(addr) &&
1.7012 + !_bit_map->isMarked(addr)) {
1.7013 + // mark bit map (object is now grey)
1.7014 + _bit_map->mark(addr);
1.7015 + // push on marking stack (stack should be empty), and drain the
1.7016 + // stack by applying this closure to the oops in the oops popped
1.7017 + // from the stack (i.e. blacken the grey objects)
1.7018 + bool res = _mark_stack->push(obj);
1.7019 + assert(res, "Should have space to push on empty stack");
1.7020 + do {
1.7021 + oop new_oop = _mark_stack->pop();
1.7022 + assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
1.7023 + assert(_bit_map->isMarked((HeapWord*)new_oop),
1.7024 + "only grey objects on this stack");
1.7025 + // iterate over the oops in this oop, marking and pushing
1.7026 + // the ones in CMS heap (i.e. in _span).
1.7027 + new_oop->oop_iterate(&_pushAndMarkClosure);
1.7028 + // check if it's time to yield
1.7029 + do_yield_check();
1.7030 + } while (!_mark_stack->isEmpty() ||
1.7031 + (!_concurrent_precleaning && take_from_overflow_list()));
1.7032 + // if marking stack is empty, and we are not doing this
1.7033 + // during precleaning, then check the overflow list
1.7034 + }
1.7035 + assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
1.7036 + assert(_collector->overflow_list_is_empty(),
1.7037 + "overflow list was drained above");
1.7038 + // We could restore evacuated mark words, if any, used for
1.7039 + // overflow list links here because the overflow list is
1.7040 + // provably empty here. That would reduce the maximum
1.7041 + // size requirements for preserved_{oop,mark}_stack.
1.7042 + // But we'll just postpone it until we are all done
1.7043 + // so we can just stream through.
1.7044 + if (!_concurrent_precleaning && CMSOverflowEarlyRestoration) {
1.7045 + _collector->restore_preserved_marks_if_any();
1.7046 + assert(_collector->no_preserved_marks(), "No preserved marks");
1.7047 + }
1.7048 + assert(!CMSOverflowEarlyRestoration || _collector->no_preserved_marks(),
1.7049 + "All preserved marks should have been restored above");
1.7050 + }
1.7051 +}
1.7052 +
1.7053 +void MarkRefsIntoAndScanClosure::do_oop(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
1.7054 +void MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
1.7055 +
1.7056 +void MarkRefsIntoAndScanClosure::do_yield_work() {
1.7057 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.7058 + "CMS thread should hold CMS token");
1.7059 + assert_lock_strong(_freelistLock);
1.7060 + assert_lock_strong(_bit_map->lock());
1.7061 + // relinquish the free_list_lock and bitMaplock()
1.7062 + _bit_map->lock()->unlock();
1.7063 + _freelistLock->unlock();
1.7064 + ConcurrentMarkSweepThread::desynchronize(true);
1.7065 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.7066 + _collector->stopTimer();
1.7067 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.7068 + if (PrintCMSStatistics != 0) {
1.7069 + _collector->incrementYields();
1.7070 + }
1.7071 + _collector->icms_wait();
1.7072 +
1.7073 + // See the comment in coordinator_yield()
1.7074 + for (unsigned i = 0;
1.7075 + i < CMSYieldSleepCount &&
1.7076 + ConcurrentMarkSweepThread::should_yield() &&
1.7077 + !CMSCollector::foregroundGCIsActive();
1.7078 + ++i) {
1.7079 + os::sleep(Thread::current(), 1, false);
1.7080 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.7081 + }
1.7082 +
1.7083 + ConcurrentMarkSweepThread::synchronize(true);
1.7084 + _freelistLock->lock_without_safepoint_check();
1.7085 + _bit_map->lock()->lock_without_safepoint_check();
1.7086 + _collector->startTimer();
1.7087 +}
1.7088 +
1.7089 +///////////////////////////////////////////////////////////
1.7090 +// Par_MarkRefsIntoAndScanClosure: a parallel version of
1.7091 +// MarkRefsIntoAndScanClosure
1.7092 +///////////////////////////////////////////////////////////
1.7093 +Par_MarkRefsIntoAndScanClosure::Par_MarkRefsIntoAndScanClosure(
1.7094 + CMSCollector* collector, MemRegion span, ReferenceProcessor* rp,
1.7095 + CMSBitMap* bit_map, OopTaskQueue* work_queue):
1.7096 + _span(span),
1.7097 + _bit_map(bit_map),
1.7098 + _work_queue(work_queue),
1.7099 + _low_water_mark(MIN2((uint)(work_queue->max_elems()/4),
1.7100 + (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads))),
1.7101 + _par_pushAndMarkClosure(collector, span, rp, bit_map, work_queue)
1.7102 +{
1.7103 + _ref_processor = rp;
1.7104 + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
1.7105 +}
1.7106 +
1.7107 +// This closure is used to mark refs into the CMS generation at the
1.7108 +// second (final) checkpoint, and to scan and transitively follow
1.7109 +// the unmarked oops. The marks are made in the marking bit map and
1.7110 +// the work_queue is used for keeping the (newly) grey objects during
1.7111 +// the scan phase whence they are also available for stealing by parallel
1.7112 +// threads. Since the marking bit map is shared, updates are
1.7113 +// synchronized (via CAS).
1.7114 +void Par_MarkRefsIntoAndScanClosure::do_oop(oop obj) {
1.7115 + if (obj != NULL) {
1.7116 + // Ignore mark word because this could be an already marked oop
1.7117 + // that may be chained at the end of the overflow list.
1.7118 + assert(obj->is_oop(true), "expected an oop");
1.7119 + HeapWord* addr = (HeapWord*)obj;
1.7120 + if (_span.contains(addr) &&
1.7121 + !_bit_map->isMarked(addr)) {
1.7122 + // mark bit map (object will become grey):
1.7123 + // It is possible for several threads to be
1.7124 + // trying to "claim" this object concurrently;
1.7125 + // the unique thread that succeeds in marking the
1.7126 + // object first will do the subsequent push on
1.7127 + // to the work queue (or overflow list).
1.7128 + if (_bit_map->par_mark(addr)) {
1.7129 + // push on work_queue (which may not be empty), and trim the
1.7130 + // queue to an appropriate length by applying this closure to
1.7131 + // the oops in the oops popped from the stack (i.e. blacken the
1.7132 + // grey objects)
1.7133 + bool res = _work_queue->push(obj);
1.7134 + assert(res, "Low water mark should be less than capacity?");
1.7135 + trim_queue(_low_water_mark);
1.7136 + } // Else, another thread claimed the object
1.7137 + }
1.7138 + }
1.7139 +}
1.7140 +
1.7141 +void Par_MarkRefsIntoAndScanClosure::do_oop(oop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
1.7142 +void Par_MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
1.7143 +
1.7144 +// This closure is used to rescan the marked objects on the dirty cards
1.7145 +// in the mod union table and the card table proper.
1.7146 +size_t ScanMarkedObjectsAgainCarefullyClosure::do_object_careful_m(
1.7147 + oop p, MemRegion mr) {
1.7148 +
1.7149 + size_t size = 0;
1.7150 + HeapWord* addr = (HeapWord*)p;
1.7151 + DEBUG_ONLY(_collector->verify_work_stacks_empty();)
1.7152 + assert(_span.contains(addr), "we are scanning the CMS generation");
1.7153 + // check if it's time to yield
1.7154 + if (do_yield_check()) {
1.7155 + // We yielded for some foreground stop-world work,
1.7156 + // and we have been asked to abort this ongoing preclean cycle.
1.7157 + return 0;
1.7158 + }
1.7159 + if (_bitMap->isMarked(addr)) {
1.7160 + // it's marked; is it potentially uninitialized?
1.7161 + if (p->klass_or_null() != NULL) {
1.7162 + // an initialized object; ignore mark word in verification below
1.7163 + // since we are running concurrent with mutators
1.7164 + assert(p->is_oop(true), "should be an oop");
1.7165 + if (p->is_objArray()) {
1.7166 + // objArrays are precisely marked; restrict scanning
1.7167 + // to dirty cards only.
1.7168 + size = CompactibleFreeListSpace::adjustObjectSize(
1.7169 + p->oop_iterate(_scanningClosure, mr));
1.7170 + } else {
1.7171 + // A non-array may have been imprecisely marked; we need
1.7172 + // to scan object in its entirety.
1.7173 + size = CompactibleFreeListSpace::adjustObjectSize(
1.7174 + p->oop_iterate(_scanningClosure));
1.7175 + }
1.7176 + #ifdef ASSERT
1.7177 + size_t direct_size =
1.7178 + CompactibleFreeListSpace::adjustObjectSize(p->size());
1.7179 + assert(size == direct_size, "Inconsistency in size");
1.7180 + assert(size >= 3, "Necessary for Printezis marks to work");
1.7181 + if (!_bitMap->isMarked(addr+1)) {
1.7182 + _bitMap->verifyNoOneBitsInRange(addr+2, addr+size);
1.7183 + } else {
1.7184 + _bitMap->verifyNoOneBitsInRange(addr+2, addr+size-1);
1.7185 + assert(_bitMap->isMarked(addr+size-1),
1.7186 + "inconsistent Printezis mark");
1.7187 + }
1.7188 + #endif // ASSERT
1.7189 + } else {
1.7190 + // an unitialized object
1.7191 + assert(_bitMap->isMarked(addr+1), "missing Printezis mark?");
1.7192 + HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
1.7193 + size = pointer_delta(nextOneAddr + 1, addr);
1.7194 + assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
1.7195 + "alignment problem");
1.7196 + // Note that pre-cleaning needn't redirty the card. OopDesc::set_klass()
1.7197 + // will dirty the card when the klass pointer is installed in the
1.7198 + // object (signalling the completion of initialization).
1.7199 + }
1.7200 + } else {
1.7201 + // Either a not yet marked object or an uninitialized object
1.7202 + if (p->klass_or_null() == NULL) {
1.7203 + // An uninitialized object, skip to the next card, since
1.7204 + // we may not be able to read its P-bits yet.
1.7205 + assert(size == 0, "Initial value");
1.7206 + } else {
1.7207 + // An object not (yet) reached by marking: we merely need to
1.7208 + // compute its size so as to go look at the next block.
1.7209 + assert(p->is_oop(true), "should be an oop");
1.7210 + size = CompactibleFreeListSpace::adjustObjectSize(p->size());
1.7211 + }
1.7212 + }
1.7213 + DEBUG_ONLY(_collector->verify_work_stacks_empty();)
1.7214 + return size;
1.7215 +}
1.7216 +
1.7217 +void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
1.7218 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.7219 + "CMS thread should hold CMS token");
1.7220 + assert_lock_strong(_freelistLock);
1.7221 + assert_lock_strong(_bitMap->lock());
1.7222 + // relinquish the free_list_lock and bitMaplock()
1.7223 + _bitMap->lock()->unlock();
1.7224 + _freelistLock->unlock();
1.7225 + ConcurrentMarkSweepThread::desynchronize(true);
1.7226 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.7227 + _collector->stopTimer();
1.7228 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.7229 + if (PrintCMSStatistics != 0) {
1.7230 + _collector->incrementYields();
1.7231 + }
1.7232 + _collector->icms_wait();
1.7233 +
1.7234 + // See the comment in coordinator_yield()
1.7235 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.7236 + ConcurrentMarkSweepThread::should_yield() &&
1.7237 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.7238 + os::sleep(Thread::current(), 1, false);
1.7239 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.7240 + }
1.7241 +
1.7242 + ConcurrentMarkSweepThread::synchronize(true);
1.7243 + _freelistLock->lock_without_safepoint_check();
1.7244 + _bitMap->lock()->lock_without_safepoint_check();
1.7245 + _collector->startTimer();
1.7246 +}
1.7247 +
1.7248 +
1.7249 +//////////////////////////////////////////////////////////////////
1.7250 +// SurvivorSpacePrecleanClosure
1.7251 +//////////////////////////////////////////////////////////////////
1.7252 +// This (single-threaded) closure is used to preclean the oops in
1.7253 +// the survivor spaces.
1.7254 +size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) {
1.7255 +
1.7256 + HeapWord* addr = (HeapWord*)p;
1.7257 + DEBUG_ONLY(_collector->verify_work_stacks_empty();)
1.7258 + assert(!_span.contains(addr), "we are scanning the survivor spaces");
1.7259 + assert(p->klass_or_null() != NULL, "object should be initializd");
1.7260 + // an initialized object; ignore mark word in verification below
1.7261 + // since we are running concurrent with mutators
1.7262 + assert(p->is_oop(true), "should be an oop");
1.7263 + // Note that we do not yield while we iterate over
1.7264 + // the interior oops of p, pushing the relevant ones
1.7265 + // on our marking stack.
1.7266 + size_t size = p->oop_iterate(_scanning_closure);
1.7267 + do_yield_check();
1.7268 + // Observe that below, we do not abandon the preclean
1.7269 + // phase as soon as we should; rather we empty the
1.7270 + // marking stack before returning. This is to satisfy
1.7271 + // some existing assertions. In general, it may be a
1.7272 + // good idea to abort immediately and complete the marking
1.7273 + // from the grey objects at a later time.
1.7274 + while (!_mark_stack->isEmpty()) {
1.7275 + oop new_oop = _mark_stack->pop();
1.7276 + assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
1.7277 + assert(_bit_map->isMarked((HeapWord*)new_oop),
1.7278 + "only grey objects on this stack");
1.7279 + // iterate over the oops in this oop, marking and pushing
1.7280 + // the ones in CMS heap (i.e. in _span).
1.7281 + new_oop->oop_iterate(_scanning_closure);
1.7282 + // check if it's time to yield
1.7283 + do_yield_check();
1.7284 + }
1.7285 + unsigned int after_count =
1.7286 + GenCollectedHeap::heap()->total_collections();
1.7287 + bool abort = (_before_count != after_count) ||
1.7288 + _collector->should_abort_preclean();
1.7289 + return abort ? 0 : size;
1.7290 +}
1.7291 +
1.7292 +void SurvivorSpacePrecleanClosure::do_yield_work() {
1.7293 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.7294 + "CMS thread should hold CMS token");
1.7295 + assert_lock_strong(_bit_map->lock());
1.7296 + // Relinquish the bit map lock
1.7297 + _bit_map->lock()->unlock();
1.7298 + ConcurrentMarkSweepThread::desynchronize(true);
1.7299 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.7300 + _collector->stopTimer();
1.7301 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.7302 + if (PrintCMSStatistics != 0) {
1.7303 + _collector->incrementYields();
1.7304 + }
1.7305 + _collector->icms_wait();
1.7306 +
1.7307 + // See the comment in coordinator_yield()
1.7308 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.7309 + ConcurrentMarkSweepThread::should_yield() &&
1.7310 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.7311 + os::sleep(Thread::current(), 1, false);
1.7312 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.7313 + }
1.7314 +
1.7315 + ConcurrentMarkSweepThread::synchronize(true);
1.7316 + _bit_map->lock()->lock_without_safepoint_check();
1.7317 + _collector->startTimer();
1.7318 +}
1.7319 +
1.7320 +// This closure is used to rescan the marked objects on the dirty cards
1.7321 +// in the mod union table and the card table proper. In the parallel
1.7322 +// case, although the bitMap is shared, we do a single read so the
1.7323 +// isMarked() query is "safe".
1.7324 +bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) {
1.7325 + // Ignore mark word because we are running concurrent with mutators
1.7326 + assert(p->is_oop_or_null(true), "expected an oop or null");
1.7327 + HeapWord* addr = (HeapWord*)p;
1.7328 + assert(_span.contains(addr), "we are scanning the CMS generation");
1.7329 + bool is_obj_array = false;
1.7330 + #ifdef ASSERT
1.7331 + if (!_parallel) {
1.7332 + assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)");
1.7333 + assert(_collector->overflow_list_is_empty(),
1.7334 + "overflow list should be empty");
1.7335 +
1.7336 + }
1.7337 + #endif // ASSERT
1.7338 + if (_bit_map->isMarked(addr)) {
1.7339 + // Obj arrays are precisely marked, non-arrays are not;
1.7340 + // so we scan objArrays precisely and non-arrays in their
1.7341 + // entirety.
1.7342 + if (p->is_objArray()) {
1.7343 + is_obj_array = true;
1.7344 + if (_parallel) {
1.7345 + p->oop_iterate(_par_scan_closure, mr);
1.7346 + } else {
1.7347 + p->oop_iterate(_scan_closure, mr);
1.7348 + }
1.7349 + } else {
1.7350 + if (_parallel) {
1.7351 + p->oop_iterate(_par_scan_closure);
1.7352 + } else {
1.7353 + p->oop_iterate(_scan_closure);
1.7354 + }
1.7355 + }
1.7356 + }
1.7357 + #ifdef ASSERT
1.7358 + if (!_parallel) {
1.7359 + assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
1.7360 + assert(_collector->overflow_list_is_empty(),
1.7361 + "overflow list should be empty");
1.7362 +
1.7363 + }
1.7364 + #endif // ASSERT
1.7365 + return is_obj_array;
1.7366 +}
1.7367 +
1.7368 +MarkFromRootsClosure::MarkFromRootsClosure(CMSCollector* collector,
1.7369 + MemRegion span,
1.7370 + CMSBitMap* bitMap, CMSMarkStack* markStack,
1.7371 + bool should_yield, bool verifying):
1.7372 + _collector(collector),
1.7373 + _span(span),
1.7374 + _bitMap(bitMap),
1.7375 + _mut(&collector->_modUnionTable),
1.7376 + _markStack(markStack),
1.7377 + _yield(should_yield),
1.7378 + _skipBits(0)
1.7379 +{
1.7380 + assert(_markStack->isEmpty(), "stack should be empty");
1.7381 + _finger = _bitMap->startWord();
1.7382 + _threshold = _finger;
1.7383 + assert(_collector->_restart_addr == NULL, "Sanity check");
1.7384 + assert(_span.contains(_finger), "Out of bounds _finger?");
1.7385 + DEBUG_ONLY(_verifying = verifying;)
1.7386 +}
1.7387 +
1.7388 +void MarkFromRootsClosure::reset(HeapWord* addr) {
1.7389 + assert(_markStack->isEmpty(), "would cause duplicates on stack");
1.7390 + assert(_span.contains(addr), "Out of bounds _finger?");
1.7391 + _finger = addr;
1.7392 + _threshold = (HeapWord*)round_to(
1.7393 + (intptr_t)_finger, CardTableModRefBS::card_size);
1.7394 +}
1.7395 +
1.7396 +// Should revisit to see if this should be restructured for
1.7397 +// greater efficiency.
1.7398 +bool MarkFromRootsClosure::do_bit(size_t offset) {
1.7399 + if (_skipBits > 0) {
1.7400 + _skipBits--;
1.7401 + return true;
1.7402 + }
1.7403 + // convert offset into a HeapWord*
1.7404 + HeapWord* addr = _bitMap->startWord() + offset;
1.7405 + assert(_bitMap->endWord() && addr < _bitMap->endWord(),
1.7406 + "address out of range");
1.7407 + assert(_bitMap->isMarked(addr), "tautology");
1.7408 + if (_bitMap->isMarked(addr+1)) {
1.7409 + // this is an allocated but not yet initialized object
1.7410 + assert(_skipBits == 0, "tautology");
1.7411 + _skipBits = 2; // skip next two marked bits ("Printezis-marks")
1.7412 + oop p = oop(addr);
1.7413 + if (p->klass_or_null() == NULL) {
1.7414 + DEBUG_ONLY(if (!_verifying) {)
1.7415 + // We re-dirty the cards on which this object lies and increase
1.7416 + // the _threshold so that we'll come back to scan this object
1.7417 + // during the preclean or remark phase. (CMSCleanOnEnter)
1.7418 + if (CMSCleanOnEnter) {
1.7419 + size_t sz = _collector->block_size_using_printezis_bits(addr);
1.7420 + HeapWord* end_card_addr = (HeapWord*)round_to(
1.7421 + (intptr_t)(addr+sz), CardTableModRefBS::card_size);
1.7422 + MemRegion redirty_range = MemRegion(addr, end_card_addr);
1.7423 + assert(!redirty_range.is_empty(), "Arithmetical tautology");
1.7424 + // Bump _threshold to end_card_addr; note that
1.7425 + // _threshold cannot possibly exceed end_card_addr, anyhow.
1.7426 + // This prevents future clearing of the card as the scan proceeds
1.7427 + // to the right.
1.7428 + assert(_threshold <= end_card_addr,
1.7429 + "Because we are just scanning into this object");
1.7430 + if (_threshold < end_card_addr) {
1.7431 + _threshold = end_card_addr;
1.7432 + }
1.7433 + if (p->klass_or_null() != NULL) {
1.7434 + // Redirty the range of cards...
1.7435 + _mut->mark_range(redirty_range);
1.7436 + } // ...else the setting of klass will dirty the card anyway.
1.7437 + }
1.7438 + DEBUG_ONLY(})
1.7439 + return true;
1.7440 + }
1.7441 + }
1.7442 + scanOopsInOop(addr);
1.7443 + return true;
1.7444 +}
1.7445 +
1.7446 +// We take a break if we've been at this for a while,
1.7447 +// so as to avoid monopolizing the locks involved.
1.7448 +void MarkFromRootsClosure::do_yield_work() {
1.7449 + // First give up the locks, then yield, then re-lock
1.7450 + // We should probably use a constructor/destructor idiom to
1.7451 + // do this unlock/lock or modify the MutexUnlocker class to
1.7452 + // serve our purpose. XXX
1.7453 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.7454 + "CMS thread should hold CMS token");
1.7455 + assert_lock_strong(_bitMap->lock());
1.7456 + _bitMap->lock()->unlock();
1.7457 + ConcurrentMarkSweepThread::desynchronize(true);
1.7458 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.7459 + _collector->stopTimer();
1.7460 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.7461 + if (PrintCMSStatistics != 0) {
1.7462 + _collector->incrementYields();
1.7463 + }
1.7464 + _collector->icms_wait();
1.7465 +
1.7466 + // See the comment in coordinator_yield()
1.7467 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.7468 + ConcurrentMarkSweepThread::should_yield() &&
1.7469 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.7470 + os::sleep(Thread::current(), 1, false);
1.7471 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.7472 + }
1.7473 +
1.7474 + ConcurrentMarkSweepThread::synchronize(true);
1.7475 + _bitMap->lock()->lock_without_safepoint_check();
1.7476 + _collector->startTimer();
1.7477 +}
1.7478 +
1.7479 +void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) {
1.7480 + assert(_bitMap->isMarked(ptr), "expected bit to be set");
1.7481 + assert(_markStack->isEmpty(),
1.7482 + "should drain stack to limit stack usage");
1.7483 + // convert ptr to an oop preparatory to scanning
1.7484 + oop obj = oop(ptr);
1.7485 + // Ignore mark word in verification below, since we
1.7486 + // may be running concurrent with mutators.
1.7487 + assert(obj->is_oop(true), "should be an oop");
1.7488 + assert(_finger <= ptr, "_finger runneth ahead");
1.7489 + // advance the finger to right end of this object
1.7490 + _finger = ptr + obj->size();
1.7491 + assert(_finger > ptr, "we just incremented it above");
1.7492 + // On large heaps, it may take us some time to get through
1.7493 + // the marking phase (especially if running iCMS). During
1.7494 + // this time it's possible that a lot of mutations have
1.7495 + // accumulated in the card table and the mod union table --
1.7496 + // these mutation records are redundant until we have
1.7497 + // actually traced into the corresponding card.
1.7498 + // Here, we check whether advancing the finger would make
1.7499 + // us cross into a new card, and if so clear corresponding
1.7500 + // cards in the MUT (preclean them in the card-table in the
1.7501 + // future).
1.7502 +
1.7503 + DEBUG_ONLY(if (!_verifying) {)
1.7504 + // The clean-on-enter optimization is disabled by default,
1.7505 + // until we fix 6178663.
1.7506 + if (CMSCleanOnEnter && (_finger > _threshold)) {
1.7507 + // [_threshold, _finger) represents the interval
1.7508 + // of cards to be cleared in MUT (or precleaned in card table).
1.7509 + // The set of cards to be cleared is all those that overlap
1.7510 + // with the interval [_threshold, _finger); note that
1.7511 + // _threshold is always kept card-aligned but _finger isn't
1.7512 + // always card-aligned.
1.7513 + HeapWord* old_threshold = _threshold;
1.7514 + assert(old_threshold == (HeapWord*)round_to(
1.7515 + (intptr_t)old_threshold, CardTableModRefBS::card_size),
1.7516 + "_threshold should always be card-aligned");
1.7517 + _threshold = (HeapWord*)round_to(
1.7518 + (intptr_t)_finger, CardTableModRefBS::card_size);
1.7519 + MemRegion mr(old_threshold, _threshold);
1.7520 + assert(!mr.is_empty(), "Control point invariant");
1.7521 + assert(_span.contains(mr), "Should clear within span");
1.7522 + _mut->clear_range(mr);
1.7523 + }
1.7524 + DEBUG_ONLY(})
1.7525 + // Note: the finger doesn't advance while we drain
1.7526 + // the stack below.
1.7527 + PushOrMarkClosure pushOrMarkClosure(_collector,
1.7528 + _span, _bitMap, _markStack,
1.7529 + _finger, this);
1.7530 + bool res = _markStack->push(obj);
1.7531 + assert(res, "Empty non-zero size stack should have space for single push");
1.7532 + while (!_markStack->isEmpty()) {
1.7533 + oop new_oop = _markStack->pop();
1.7534 + // Skip verifying header mark word below because we are
1.7535 + // running concurrent with mutators.
1.7536 + assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
1.7537 + // now scan this oop's oops
1.7538 + new_oop->oop_iterate(&pushOrMarkClosure);
1.7539 + do_yield_check();
1.7540 + }
1.7541 + assert(_markStack->isEmpty(), "tautology, emphasizing post-condition");
1.7542 +}
1.7543 +
1.7544 +Par_MarkFromRootsClosure::Par_MarkFromRootsClosure(CMSConcMarkingTask* task,
1.7545 + CMSCollector* collector, MemRegion span,
1.7546 + CMSBitMap* bit_map,
1.7547 + OopTaskQueue* work_queue,
1.7548 + CMSMarkStack* overflow_stack,
1.7549 + bool should_yield):
1.7550 + _collector(collector),
1.7551 + _whole_span(collector->_span),
1.7552 + _span(span),
1.7553 + _bit_map(bit_map),
1.7554 + _mut(&collector->_modUnionTable),
1.7555 + _work_queue(work_queue),
1.7556 + _overflow_stack(overflow_stack),
1.7557 + _yield(should_yield),
1.7558 + _skip_bits(0),
1.7559 + _task(task)
1.7560 +{
1.7561 + assert(_work_queue->size() == 0, "work_queue should be empty");
1.7562 + _finger = span.start();
1.7563 + _threshold = _finger; // XXX Defer clear-on-enter optimization for now
1.7564 + assert(_span.contains(_finger), "Out of bounds _finger?");
1.7565 +}
1.7566 +
1.7567 +// Should revisit to see if this should be restructured for
1.7568 +// greater efficiency.
1.7569 +bool Par_MarkFromRootsClosure::do_bit(size_t offset) {
1.7570 + if (_skip_bits > 0) {
1.7571 + _skip_bits--;
1.7572 + return true;
1.7573 + }
1.7574 + // convert offset into a HeapWord*
1.7575 + HeapWord* addr = _bit_map->startWord() + offset;
1.7576 + assert(_bit_map->endWord() && addr < _bit_map->endWord(),
1.7577 + "address out of range");
1.7578 + assert(_bit_map->isMarked(addr), "tautology");
1.7579 + if (_bit_map->isMarked(addr+1)) {
1.7580 + // this is an allocated object that might not yet be initialized
1.7581 + assert(_skip_bits == 0, "tautology");
1.7582 + _skip_bits = 2; // skip next two marked bits ("Printezis-marks")
1.7583 + oop p = oop(addr);
1.7584 + if (p->klass_or_null() == NULL) {
1.7585 + // in the case of Clean-on-Enter optimization, redirty card
1.7586 + // and avoid clearing card by increasing the threshold.
1.7587 + return true;
1.7588 + }
1.7589 + }
1.7590 + scan_oops_in_oop(addr);
1.7591 + return true;
1.7592 +}
1.7593 +
1.7594 +void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) {
1.7595 + assert(_bit_map->isMarked(ptr), "expected bit to be set");
1.7596 + // Should we assert that our work queue is empty or
1.7597 + // below some drain limit?
1.7598 + assert(_work_queue->size() == 0,
1.7599 + "should drain stack to limit stack usage");
1.7600 + // convert ptr to an oop preparatory to scanning
1.7601 + oop obj = oop(ptr);
1.7602 + // Ignore mark word in verification below, since we
1.7603 + // may be running concurrent with mutators.
1.7604 + assert(obj->is_oop(true), "should be an oop");
1.7605 + assert(_finger <= ptr, "_finger runneth ahead");
1.7606 + // advance the finger to right end of this object
1.7607 + _finger = ptr + obj->size();
1.7608 + assert(_finger > ptr, "we just incremented it above");
1.7609 + // On large heaps, it may take us some time to get through
1.7610 + // the marking phase (especially if running iCMS). During
1.7611 + // this time it's possible that a lot of mutations have
1.7612 + // accumulated in the card table and the mod union table --
1.7613 + // these mutation records are redundant until we have
1.7614 + // actually traced into the corresponding card.
1.7615 + // Here, we check whether advancing the finger would make
1.7616 + // us cross into a new card, and if so clear corresponding
1.7617 + // cards in the MUT (preclean them in the card-table in the
1.7618 + // future).
1.7619 +
1.7620 + // The clean-on-enter optimization is disabled by default,
1.7621 + // until we fix 6178663.
1.7622 + if (CMSCleanOnEnter && (_finger > _threshold)) {
1.7623 + // [_threshold, _finger) represents the interval
1.7624 + // of cards to be cleared in MUT (or precleaned in card table).
1.7625 + // The set of cards to be cleared is all those that overlap
1.7626 + // with the interval [_threshold, _finger); note that
1.7627 + // _threshold is always kept card-aligned but _finger isn't
1.7628 + // always card-aligned.
1.7629 + HeapWord* old_threshold = _threshold;
1.7630 + assert(old_threshold == (HeapWord*)round_to(
1.7631 + (intptr_t)old_threshold, CardTableModRefBS::card_size),
1.7632 + "_threshold should always be card-aligned");
1.7633 + _threshold = (HeapWord*)round_to(
1.7634 + (intptr_t)_finger, CardTableModRefBS::card_size);
1.7635 + MemRegion mr(old_threshold, _threshold);
1.7636 + assert(!mr.is_empty(), "Control point invariant");
1.7637 + assert(_span.contains(mr), "Should clear within span"); // _whole_span ??
1.7638 + _mut->clear_range(mr);
1.7639 + }
1.7640 +
1.7641 + // Note: the local finger doesn't advance while we drain
1.7642 + // the stack below, but the global finger sure can and will.
1.7643 + HeapWord** gfa = _task->global_finger_addr();
1.7644 + Par_PushOrMarkClosure pushOrMarkClosure(_collector,
1.7645 + _span, _bit_map,
1.7646 + _work_queue,
1.7647 + _overflow_stack,
1.7648 + _finger,
1.7649 + gfa, this);
1.7650 + bool res = _work_queue->push(obj); // overflow could occur here
1.7651 + assert(res, "Will hold once we use workqueues");
1.7652 + while (true) {
1.7653 + oop new_oop;
1.7654 + if (!_work_queue->pop_local(new_oop)) {
1.7655 + // We emptied our work_queue; check if there's stuff that can
1.7656 + // be gotten from the overflow stack.
1.7657 + if (CMSConcMarkingTask::get_work_from_overflow_stack(
1.7658 + _overflow_stack, _work_queue)) {
1.7659 + do_yield_check();
1.7660 + continue;
1.7661 + } else { // done
1.7662 + break;
1.7663 + }
1.7664 + }
1.7665 + // Skip verifying header mark word below because we are
1.7666 + // running concurrent with mutators.
1.7667 + assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
1.7668 + // now scan this oop's oops
1.7669 + new_oop->oop_iterate(&pushOrMarkClosure);
1.7670 + do_yield_check();
1.7671 + }
1.7672 + assert(_work_queue->size() == 0, "tautology, emphasizing post-condition");
1.7673 +}
1.7674 +
1.7675 +// Yield in response to a request from VM Thread or
1.7676 +// from mutators.
1.7677 +void Par_MarkFromRootsClosure::do_yield_work() {
1.7678 + assert(_task != NULL, "sanity");
1.7679 + _task->yield();
1.7680 +}
1.7681 +
1.7682 +// A variant of the above used for verifying CMS marking work.
1.7683 +MarkFromRootsVerifyClosure::MarkFromRootsVerifyClosure(CMSCollector* collector,
1.7684 + MemRegion span,
1.7685 + CMSBitMap* verification_bm, CMSBitMap* cms_bm,
1.7686 + CMSMarkStack* mark_stack):
1.7687 + _collector(collector),
1.7688 + _span(span),
1.7689 + _verification_bm(verification_bm),
1.7690 + _cms_bm(cms_bm),
1.7691 + _mark_stack(mark_stack),
1.7692 + _pam_verify_closure(collector, span, verification_bm, cms_bm,
1.7693 + mark_stack)
1.7694 +{
1.7695 + assert(_mark_stack->isEmpty(), "stack should be empty");
1.7696 + _finger = _verification_bm->startWord();
1.7697 + assert(_collector->_restart_addr == NULL, "Sanity check");
1.7698 + assert(_span.contains(_finger), "Out of bounds _finger?");
1.7699 +}
1.7700 +
1.7701 +void MarkFromRootsVerifyClosure::reset(HeapWord* addr) {
1.7702 + assert(_mark_stack->isEmpty(), "would cause duplicates on stack");
1.7703 + assert(_span.contains(addr), "Out of bounds _finger?");
1.7704 + _finger = addr;
1.7705 +}
1.7706 +
1.7707 +// Should revisit to see if this should be restructured for
1.7708 +// greater efficiency.
1.7709 +bool MarkFromRootsVerifyClosure::do_bit(size_t offset) {
1.7710 + // convert offset into a HeapWord*
1.7711 + HeapWord* addr = _verification_bm->startWord() + offset;
1.7712 + assert(_verification_bm->endWord() && addr < _verification_bm->endWord(),
1.7713 + "address out of range");
1.7714 + assert(_verification_bm->isMarked(addr), "tautology");
1.7715 + assert(_cms_bm->isMarked(addr), "tautology");
1.7716 +
1.7717 + assert(_mark_stack->isEmpty(),
1.7718 + "should drain stack to limit stack usage");
1.7719 + // convert addr to an oop preparatory to scanning
1.7720 + oop obj = oop(addr);
1.7721 + assert(obj->is_oop(), "should be an oop");
1.7722 + assert(_finger <= addr, "_finger runneth ahead");
1.7723 + // advance the finger to right end of this object
1.7724 + _finger = addr + obj->size();
1.7725 + assert(_finger > addr, "we just incremented it above");
1.7726 + // Note: the finger doesn't advance while we drain
1.7727 + // the stack below.
1.7728 + bool res = _mark_stack->push(obj);
1.7729 + assert(res, "Empty non-zero size stack should have space for single push");
1.7730 + while (!_mark_stack->isEmpty()) {
1.7731 + oop new_oop = _mark_stack->pop();
1.7732 + assert(new_oop->is_oop(), "Oops! expected to pop an oop");
1.7733 + // now scan this oop's oops
1.7734 + new_oop->oop_iterate(&_pam_verify_closure);
1.7735 + }
1.7736 + assert(_mark_stack->isEmpty(), "tautology, emphasizing post-condition");
1.7737 + return true;
1.7738 +}
1.7739 +
1.7740 +PushAndMarkVerifyClosure::PushAndMarkVerifyClosure(
1.7741 + CMSCollector* collector, MemRegion span,
1.7742 + CMSBitMap* verification_bm, CMSBitMap* cms_bm,
1.7743 + CMSMarkStack* mark_stack):
1.7744 + CMSOopClosure(collector->ref_processor()),
1.7745 + _collector(collector),
1.7746 + _span(span),
1.7747 + _verification_bm(verification_bm),
1.7748 + _cms_bm(cms_bm),
1.7749 + _mark_stack(mark_stack)
1.7750 +{ }
1.7751 +
1.7752 +void PushAndMarkVerifyClosure::do_oop(oop* p) { PushAndMarkVerifyClosure::do_oop_work(p); }
1.7753 +void PushAndMarkVerifyClosure::do_oop(narrowOop* p) { PushAndMarkVerifyClosure::do_oop_work(p); }
1.7754 +
1.7755 +// Upon stack overflow, we discard (part of) the stack,
1.7756 +// remembering the least address amongst those discarded
1.7757 +// in CMSCollector's _restart_address.
1.7758 +void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
1.7759 + // Remember the least grey address discarded
1.7760 + HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
1.7761 + _collector->lower_restart_addr(ra);
1.7762 + _mark_stack->reset(); // discard stack contents
1.7763 + _mark_stack->expand(); // expand the stack if possible
1.7764 +}
1.7765 +
1.7766 +void PushAndMarkVerifyClosure::do_oop(oop obj) {
1.7767 + assert(obj->is_oop_or_null(), "expected an oop or NULL");
1.7768 + HeapWord* addr = (HeapWord*)obj;
1.7769 + if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
1.7770 + // Oop lies in _span and isn't yet grey or black
1.7771 + _verification_bm->mark(addr); // now grey
1.7772 + if (!_cms_bm->isMarked(addr)) {
1.7773 + oop(addr)->print();
1.7774 + gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)",
1.7775 + addr);
1.7776 + fatal("... aborting");
1.7777 + }
1.7778 +
1.7779 + if (!_mark_stack->push(obj)) { // stack overflow
1.7780 + if (PrintCMSStatistics != 0) {
1.7781 + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
1.7782 + SIZE_FORMAT, _mark_stack->capacity());
1.7783 + }
1.7784 + assert(_mark_stack->isFull(), "Else push should have succeeded");
1.7785 + handle_stack_overflow(addr);
1.7786 + }
1.7787 + // anything including and to the right of _finger
1.7788 + // will be scanned as we iterate over the remainder of the
1.7789 + // bit map
1.7790 + }
1.7791 +}
1.7792 +
1.7793 +PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
1.7794 + MemRegion span,
1.7795 + CMSBitMap* bitMap, CMSMarkStack* markStack,
1.7796 + HeapWord* finger, MarkFromRootsClosure* parent) :
1.7797 + CMSOopClosure(collector->ref_processor()),
1.7798 + _collector(collector),
1.7799 + _span(span),
1.7800 + _bitMap(bitMap),
1.7801 + _markStack(markStack),
1.7802 + _finger(finger),
1.7803 + _parent(parent)
1.7804 +{ }
1.7805 +
1.7806 +Par_PushOrMarkClosure::Par_PushOrMarkClosure(CMSCollector* collector,
1.7807 + MemRegion span,
1.7808 + CMSBitMap* bit_map,
1.7809 + OopTaskQueue* work_queue,
1.7810 + CMSMarkStack* overflow_stack,
1.7811 + HeapWord* finger,
1.7812 + HeapWord** global_finger_addr,
1.7813 + Par_MarkFromRootsClosure* parent) :
1.7814 + CMSOopClosure(collector->ref_processor()),
1.7815 + _collector(collector),
1.7816 + _whole_span(collector->_span),
1.7817 + _span(span),
1.7818 + _bit_map(bit_map),
1.7819 + _work_queue(work_queue),
1.7820 + _overflow_stack(overflow_stack),
1.7821 + _finger(finger),
1.7822 + _global_finger_addr(global_finger_addr),
1.7823 + _parent(parent)
1.7824 +{ }
1.7825 +
1.7826 +// Assumes thread-safe access by callers, who are
1.7827 +// responsible for mutual exclusion.
1.7828 +void CMSCollector::lower_restart_addr(HeapWord* low) {
1.7829 + assert(_span.contains(low), "Out of bounds addr");
1.7830 + if (_restart_addr == NULL) {
1.7831 + _restart_addr = low;
1.7832 + } else {
1.7833 + _restart_addr = MIN2(_restart_addr, low);
1.7834 + }
1.7835 +}
1.7836 +
1.7837 +// Upon stack overflow, we discard (part of) the stack,
1.7838 +// remembering the least address amongst those discarded
1.7839 +// in CMSCollector's _restart_address.
1.7840 +void PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
1.7841 + // Remember the least grey address discarded
1.7842 + HeapWord* ra = (HeapWord*)_markStack->least_value(lost);
1.7843 + _collector->lower_restart_addr(ra);
1.7844 + _markStack->reset(); // discard stack contents
1.7845 + _markStack->expand(); // expand the stack if possible
1.7846 +}
1.7847 +
1.7848 +// Upon stack overflow, we discard (part of) the stack,
1.7849 +// remembering the least address amongst those discarded
1.7850 +// in CMSCollector's _restart_address.
1.7851 +void Par_PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
1.7852 + // We need to do this under a mutex to prevent other
1.7853 + // workers from interfering with the work done below.
1.7854 + MutexLockerEx ml(_overflow_stack->par_lock(),
1.7855 + Mutex::_no_safepoint_check_flag);
1.7856 + // Remember the least grey address discarded
1.7857 + HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost);
1.7858 + _collector->lower_restart_addr(ra);
1.7859 + _overflow_stack->reset(); // discard stack contents
1.7860 + _overflow_stack->expand(); // expand the stack if possible
1.7861 +}
1.7862 +
1.7863 +void CMKlassClosure::do_klass(Klass* k) {
1.7864 + assert(_oop_closure != NULL, "Not initialized?");
1.7865 + k->oops_do(_oop_closure);
1.7866 +}
1.7867 +
1.7868 +void PushOrMarkClosure::do_oop(oop obj) {
1.7869 + // Ignore mark word because we are running concurrent with mutators.
1.7870 + assert(obj->is_oop_or_null(true), "expected an oop or NULL");
1.7871 + HeapWord* addr = (HeapWord*)obj;
1.7872 + if (_span.contains(addr) && !_bitMap->isMarked(addr)) {
1.7873 + // Oop lies in _span and isn't yet grey or black
1.7874 + _bitMap->mark(addr); // now grey
1.7875 + if (addr < _finger) {
1.7876 + // the bit map iteration has already either passed, or
1.7877 + // sampled, this bit in the bit map; we'll need to
1.7878 + // use the marking stack to scan this oop's oops.
1.7879 + bool simulate_overflow = false;
1.7880 + NOT_PRODUCT(
1.7881 + if (CMSMarkStackOverflowALot &&
1.7882 + _collector->simulate_overflow()) {
1.7883 + // simulate a stack overflow
1.7884 + simulate_overflow = true;
1.7885 + }
1.7886 + )
1.7887 + if (simulate_overflow || !_markStack->push(obj)) { // stack overflow
1.7888 + if (PrintCMSStatistics != 0) {
1.7889 + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
1.7890 + SIZE_FORMAT, _markStack->capacity());
1.7891 + }
1.7892 + assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded");
1.7893 + handle_stack_overflow(addr);
1.7894 + }
1.7895 + }
1.7896 + // anything including and to the right of _finger
1.7897 + // will be scanned as we iterate over the remainder of the
1.7898 + // bit map
1.7899 + do_yield_check();
1.7900 + }
1.7901 +}
1.7902 +
1.7903 +void PushOrMarkClosure::do_oop(oop* p) { PushOrMarkClosure::do_oop_work(p); }
1.7904 +void PushOrMarkClosure::do_oop(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); }
1.7905 +
1.7906 +void Par_PushOrMarkClosure::do_oop(oop obj) {
1.7907 + // Ignore mark word because we are running concurrent with mutators.
1.7908 + assert(obj->is_oop_or_null(true), "expected an oop or NULL");
1.7909 + HeapWord* addr = (HeapWord*)obj;
1.7910 + if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) {
1.7911 + // Oop lies in _span and isn't yet grey or black
1.7912 + // We read the global_finger (volatile read) strictly after marking oop
1.7913 + bool res = _bit_map->par_mark(addr); // now grey
1.7914 + volatile HeapWord** gfa = (volatile HeapWord**)_global_finger_addr;
1.7915 + // Should we push this marked oop on our stack?
1.7916 + // -- if someone else marked it, nothing to do
1.7917 + // -- if target oop is above global finger nothing to do
1.7918 + // -- if target oop is in chunk and above local finger
1.7919 + // then nothing to do
1.7920 + // -- else push on work queue
1.7921 + if ( !res // someone else marked it, they will deal with it
1.7922 + || (addr >= *gfa) // will be scanned in a later task
1.7923 + || (_span.contains(addr) && addr >= _finger)) { // later in this chunk
1.7924 + return;
1.7925 + }
1.7926 + // the bit map iteration has already either passed, or
1.7927 + // sampled, this bit in the bit map; we'll need to
1.7928 + // use the marking stack to scan this oop's oops.
1.7929 + bool simulate_overflow = false;
1.7930 + NOT_PRODUCT(
1.7931 + if (CMSMarkStackOverflowALot &&
1.7932 + _collector->simulate_overflow()) {
1.7933 + // simulate a stack overflow
1.7934 + simulate_overflow = true;
1.7935 + }
1.7936 + )
1.7937 + if (simulate_overflow ||
1.7938 + !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
1.7939 + // stack overflow
1.7940 + if (PrintCMSStatistics != 0) {
1.7941 + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
1.7942 + SIZE_FORMAT, _overflow_stack->capacity());
1.7943 + }
1.7944 + // We cannot assert that the overflow stack is full because
1.7945 + // it may have been emptied since.
1.7946 + assert(simulate_overflow ||
1.7947 + _work_queue->size() == _work_queue->max_elems(),
1.7948 + "Else push should have succeeded");
1.7949 + handle_stack_overflow(addr);
1.7950 + }
1.7951 + do_yield_check();
1.7952 + }
1.7953 +}
1.7954 +
1.7955 +void Par_PushOrMarkClosure::do_oop(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
1.7956 +void Par_PushOrMarkClosure::do_oop(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
1.7957 +
1.7958 +PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector,
1.7959 + MemRegion span,
1.7960 + ReferenceProcessor* rp,
1.7961 + CMSBitMap* bit_map,
1.7962 + CMSBitMap* mod_union_table,
1.7963 + CMSMarkStack* mark_stack,
1.7964 + bool concurrent_precleaning):
1.7965 + CMSOopClosure(rp),
1.7966 + _collector(collector),
1.7967 + _span(span),
1.7968 + _bit_map(bit_map),
1.7969 + _mod_union_table(mod_union_table),
1.7970 + _mark_stack(mark_stack),
1.7971 + _concurrent_precleaning(concurrent_precleaning)
1.7972 +{
1.7973 + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
1.7974 +}
1.7975 +
1.7976 +// Grey object rescan during pre-cleaning and second checkpoint phases --
1.7977 +// the non-parallel version (the parallel version appears further below.)
1.7978 +void PushAndMarkClosure::do_oop(oop obj) {
1.7979 + // Ignore mark word verification. If during concurrent precleaning,
1.7980 + // the object monitor may be locked. If during the checkpoint
1.7981 + // phases, the object may already have been reached by a different
1.7982 + // path and may be at the end of the global overflow list (so
1.7983 + // the mark word may be NULL).
1.7984 + assert(obj->is_oop_or_null(true /* ignore mark word */),
1.7985 + "expected an oop or NULL");
1.7986 + HeapWord* addr = (HeapWord*)obj;
1.7987 + // Check if oop points into the CMS generation
1.7988 + // and is not marked
1.7989 + if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
1.7990 + // a white object ...
1.7991 + _bit_map->mark(addr); // ... now grey
1.7992 + // push on the marking stack (grey set)
1.7993 + bool simulate_overflow = false;
1.7994 + NOT_PRODUCT(
1.7995 + if (CMSMarkStackOverflowALot &&
1.7996 + _collector->simulate_overflow()) {
1.7997 + // simulate a stack overflow
1.7998 + simulate_overflow = true;
1.7999 + }
1.8000 + )
1.8001 + if (simulate_overflow || !_mark_stack->push(obj)) {
1.8002 + if (_concurrent_precleaning) {
1.8003 + // During precleaning we can just dirty the appropriate card(s)
1.8004 + // in the mod union table, thus ensuring that the object remains
1.8005 + // in the grey set and continue. In the case of object arrays
1.8006 + // we need to dirty all of the cards that the object spans,
1.8007 + // since the rescan of object arrays will be limited to the
1.8008 + // dirty cards.
1.8009 + // Note that no one can be intefering with us in this action
1.8010 + // of dirtying the mod union table, so no locking or atomics
1.8011 + // are required.
1.8012 + if (obj->is_objArray()) {
1.8013 + size_t sz = obj->size();
1.8014 + HeapWord* end_card_addr = (HeapWord*)round_to(
1.8015 + (intptr_t)(addr+sz), CardTableModRefBS::card_size);
1.8016 + MemRegion redirty_range = MemRegion(addr, end_card_addr);
1.8017 + assert(!redirty_range.is_empty(), "Arithmetical tautology");
1.8018 + _mod_union_table->mark_range(redirty_range);
1.8019 + } else {
1.8020 + _mod_union_table->mark(addr);
1.8021 + }
1.8022 + _collector->_ser_pmc_preclean_ovflw++;
1.8023 + } else {
1.8024 + // During the remark phase, we need to remember this oop
1.8025 + // in the overflow list.
1.8026 + _collector->push_on_overflow_list(obj);
1.8027 + _collector->_ser_pmc_remark_ovflw++;
1.8028 + }
1.8029 + }
1.8030 + }
1.8031 +}
1.8032 +
1.8033 +Par_PushAndMarkClosure::Par_PushAndMarkClosure(CMSCollector* collector,
1.8034 + MemRegion span,
1.8035 + ReferenceProcessor* rp,
1.8036 + CMSBitMap* bit_map,
1.8037 + OopTaskQueue* work_queue):
1.8038 + CMSOopClosure(rp),
1.8039 + _collector(collector),
1.8040 + _span(span),
1.8041 + _bit_map(bit_map),
1.8042 + _work_queue(work_queue)
1.8043 +{
1.8044 + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
1.8045 +}
1.8046 +
1.8047 +void PushAndMarkClosure::do_oop(oop* p) { PushAndMarkClosure::do_oop_work(p); }
1.8048 +void PushAndMarkClosure::do_oop(narrowOop* p) { PushAndMarkClosure::do_oop_work(p); }
1.8049 +
1.8050 +// Grey object rescan during second checkpoint phase --
1.8051 +// the parallel version.
1.8052 +void Par_PushAndMarkClosure::do_oop(oop obj) {
1.8053 + // In the assert below, we ignore the mark word because
1.8054 + // this oop may point to an already visited object that is
1.8055 + // on the overflow stack (in which case the mark word has
1.8056 + // been hijacked for chaining into the overflow stack --
1.8057 + // if this is the last object in the overflow stack then
1.8058 + // its mark word will be NULL). Because this object may
1.8059 + // have been subsequently popped off the global overflow
1.8060 + // stack, and the mark word possibly restored to the prototypical
1.8061 + // value, by the time we get to examined this failing assert in
1.8062 + // the debugger, is_oop_or_null(false) may subsequently start
1.8063 + // to hold.
1.8064 + assert(obj->is_oop_or_null(true),
1.8065 + "expected an oop or NULL");
1.8066 + HeapWord* addr = (HeapWord*)obj;
1.8067 + // Check if oop points into the CMS generation
1.8068 + // and is not marked
1.8069 + if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
1.8070 + // a white object ...
1.8071 + // If we manage to "claim" the object, by being the
1.8072 + // first thread to mark it, then we push it on our
1.8073 + // marking stack
1.8074 + if (_bit_map->par_mark(addr)) { // ... now grey
1.8075 + // push on work queue (grey set)
1.8076 + bool simulate_overflow = false;
1.8077 + NOT_PRODUCT(
1.8078 + if (CMSMarkStackOverflowALot &&
1.8079 + _collector->par_simulate_overflow()) {
1.8080 + // simulate a stack overflow
1.8081 + simulate_overflow = true;
1.8082 + }
1.8083 + )
1.8084 + if (simulate_overflow || !_work_queue->push(obj)) {
1.8085 + _collector->par_push_on_overflow_list(obj);
1.8086 + _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS
1.8087 + }
1.8088 + } // Else, some other thread got there first
1.8089 + }
1.8090 +}
1.8091 +
1.8092 +void Par_PushAndMarkClosure::do_oop(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
1.8093 +void Par_PushAndMarkClosure::do_oop(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
1.8094 +
1.8095 +void CMSPrecleanRefsYieldClosure::do_yield_work() {
1.8096 + Mutex* bml = _collector->bitMapLock();
1.8097 + assert_lock_strong(bml);
1.8098 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.8099 + "CMS thread should hold CMS token");
1.8100 +
1.8101 + bml->unlock();
1.8102 + ConcurrentMarkSweepThread::desynchronize(true);
1.8103 +
1.8104 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.8105 +
1.8106 + _collector->stopTimer();
1.8107 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.8108 + if (PrintCMSStatistics != 0) {
1.8109 + _collector->incrementYields();
1.8110 + }
1.8111 + _collector->icms_wait();
1.8112 +
1.8113 + // See the comment in coordinator_yield()
1.8114 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.8115 + ConcurrentMarkSweepThread::should_yield() &&
1.8116 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.8117 + os::sleep(Thread::current(), 1, false);
1.8118 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.8119 + }
1.8120 +
1.8121 + ConcurrentMarkSweepThread::synchronize(true);
1.8122 + bml->lock();
1.8123 +
1.8124 + _collector->startTimer();
1.8125 +}
1.8126 +
1.8127 +bool CMSPrecleanRefsYieldClosure::should_return() {
1.8128 + if (ConcurrentMarkSweepThread::should_yield()) {
1.8129 + do_yield_work();
1.8130 + }
1.8131 + return _collector->foregroundGCIsActive();
1.8132 +}
1.8133 +
1.8134 +void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
1.8135 + assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
1.8136 + "mr should be aligned to start at a card boundary");
1.8137 + // We'd like to assert:
1.8138 + // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
1.8139 + // "mr should be a range of cards");
1.8140 + // However, that would be too strong in one case -- the last
1.8141 + // partition ends at _unallocated_block which, in general, can be
1.8142 + // an arbitrary boundary, not necessarily card aligned.
1.8143 + if (PrintCMSStatistics != 0) {
1.8144 + _num_dirty_cards +=
1.8145 + mr.word_size()/CardTableModRefBS::card_size_in_words;
1.8146 + }
1.8147 + _space->object_iterate_mem(mr, &_scan_cl);
1.8148 +}
1.8149 +
1.8150 +SweepClosure::SweepClosure(CMSCollector* collector,
1.8151 + ConcurrentMarkSweepGeneration* g,
1.8152 + CMSBitMap* bitMap, bool should_yield) :
1.8153 + _collector(collector),
1.8154 + _g(g),
1.8155 + _sp(g->cmsSpace()),
1.8156 + _limit(_sp->sweep_limit()),
1.8157 + _freelistLock(_sp->freelistLock()),
1.8158 + _bitMap(bitMap),
1.8159 + _yield(should_yield),
1.8160 + _inFreeRange(false), // No free range at beginning of sweep
1.8161 + _freeRangeInFreeLists(false), // No free range at beginning of sweep
1.8162 + _lastFreeRangeCoalesced(false),
1.8163 + _freeFinger(g->used_region().start())
1.8164 +{
1.8165 + NOT_PRODUCT(
1.8166 + _numObjectsFreed = 0;
1.8167 + _numWordsFreed = 0;
1.8168 + _numObjectsLive = 0;
1.8169 + _numWordsLive = 0;
1.8170 + _numObjectsAlreadyFree = 0;
1.8171 + _numWordsAlreadyFree = 0;
1.8172 + _last_fc = NULL;
1.8173 +
1.8174 + _sp->initializeIndexedFreeListArrayReturnedBytes();
1.8175 + _sp->dictionary()->initialize_dict_returned_bytes();
1.8176 + )
1.8177 + assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
1.8178 + "sweep _limit out of bounds");
1.8179 + if (CMSTraceSweeper) {
1.8180 + gclog_or_tty->print_cr("\n====================\nStarting new sweep with limit " PTR_FORMAT,
1.8181 + _limit);
1.8182 + }
1.8183 +}
1.8184 +
1.8185 +void SweepClosure::print_on(outputStream* st) const {
1.8186 + tty->print_cr("_sp = [" PTR_FORMAT "," PTR_FORMAT ")",
1.8187 + _sp->bottom(), _sp->end());
1.8188 + tty->print_cr("_limit = " PTR_FORMAT, _limit);
1.8189 + tty->print_cr("_freeFinger = " PTR_FORMAT, _freeFinger);
1.8190 + NOT_PRODUCT(tty->print_cr("_last_fc = " PTR_FORMAT, _last_fc);)
1.8191 + tty->print_cr("_inFreeRange = %d, _freeRangeInFreeLists = %d, _lastFreeRangeCoalesced = %d",
1.8192 + _inFreeRange, _freeRangeInFreeLists, _lastFreeRangeCoalesced);
1.8193 +}
1.8194 +
1.8195 +#ifndef PRODUCT
1.8196 +// Assertion checking only: no useful work in product mode --
1.8197 +// however, if any of the flags below become product flags,
1.8198 +// you may need to review this code to see if it needs to be
1.8199 +// enabled in product mode.
1.8200 +SweepClosure::~SweepClosure() {
1.8201 + assert_lock_strong(_freelistLock);
1.8202 + assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
1.8203 + "sweep _limit out of bounds");
1.8204 + if (inFreeRange()) {
1.8205 + warning("inFreeRange() should have been reset; dumping state of SweepClosure");
1.8206 + print();
1.8207 + ShouldNotReachHere();
1.8208 + }
1.8209 + if (Verbose && PrintGC) {
1.8210 + gclog_or_tty->print("Collected "SIZE_FORMAT" objects, " SIZE_FORMAT " bytes",
1.8211 + _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
1.8212 + gclog_or_tty->print_cr("\nLive "SIZE_FORMAT" objects, "
1.8213 + SIZE_FORMAT" bytes "
1.8214 + "Already free "SIZE_FORMAT" objects, "SIZE_FORMAT" bytes",
1.8215 + _numObjectsLive, _numWordsLive*sizeof(HeapWord),
1.8216 + _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
1.8217 + size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree)
1.8218 + * sizeof(HeapWord);
1.8219 + gclog_or_tty->print_cr("Total sweep: "SIZE_FORMAT" bytes", totalBytes);
1.8220 +
1.8221 + if (PrintCMSStatistics && CMSVerifyReturnedBytes) {
1.8222 + size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
1.8223 + size_t dict_returned_bytes = _sp->dictionary()->sum_dict_returned_bytes();
1.8224 + size_t returned_bytes = indexListReturnedBytes + dict_returned_bytes;
1.8225 + gclog_or_tty->print("Returned "SIZE_FORMAT" bytes", returned_bytes);
1.8226 + gclog_or_tty->print(" Indexed List Returned "SIZE_FORMAT" bytes",
1.8227 + indexListReturnedBytes);
1.8228 + gclog_or_tty->print_cr(" Dictionary Returned "SIZE_FORMAT" bytes",
1.8229 + dict_returned_bytes);
1.8230 + }
1.8231 + }
1.8232 + if (CMSTraceSweeper) {
1.8233 + gclog_or_tty->print_cr("end of sweep with _limit = " PTR_FORMAT "\n================",
1.8234 + _limit);
1.8235 + }
1.8236 +}
1.8237 +#endif // PRODUCT
1.8238 +
1.8239 +void SweepClosure::initialize_free_range(HeapWord* freeFinger,
1.8240 + bool freeRangeInFreeLists) {
1.8241 + if (CMSTraceSweeper) {
1.8242 + gclog_or_tty->print("---- Start free range at 0x%x with free block (%d)\n",
1.8243 + freeFinger, freeRangeInFreeLists);
1.8244 + }
1.8245 + assert(!inFreeRange(), "Trampling existing free range");
1.8246 + set_inFreeRange(true);
1.8247 + set_lastFreeRangeCoalesced(false);
1.8248 +
1.8249 + set_freeFinger(freeFinger);
1.8250 + set_freeRangeInFreeLists(freeRangeInFreeLists);
1.8251 + if (CMSTestInFreeList) {
1.8252 + if (freeRangeInFreeLists) {
1.8253 + FreeChunk* fc = (FreeChunk*) freeFinger;
1.8254 + assert(fc->is_free(), "A chunk on the free list should be free.");
1.8255 + assert(fc->size() > 0, "Free range should have a size");
1.8256 + assert(_sp->verify_chunk_in_free_list(fc), "Chunk is not in free lists");
1.8257 + }
1.8258 + }
1.8259 +}
1.8260 +
1.8261 +// Note that the sweeper runs concurrently with mutators. Thus,
1.8262 +// it is possible for direct allocation in this generation to happen
1.8263 +// in the middle of the sweep. Note that the sweeper also coalesces
1.8264 +// contiguous free blocks. Thus, unless the sweeper and the allocator
1.8265 +// synchronize appropriately freshly allocated blocks may get swept up.
1.8266 +// This is accomplished by the sweeper locking the free lists while
1.8267 +// it is sweeping. Thus blocks that are determined to be free are
1.8268 +// indeed free. There is however one additional complication:
1.8269 +// blocks that have been allocated since the final checkpoint and
1.8270 +// mark, will not have been marked and so would be treated as
1.8271 +// unreachable and swept up. To prevent this, the allocator marks
1.8272 +// the bit map when allocating during the sweep phase. This leads,
1.8273 +// however, to a further complication -- objects may have been allocated
1.8274 +// but not yet initialized -- in the sense that the header isn't yet
1.8275 +// installed. The sweeper can not then determine the size of the block
1.8276 +// in order to skip over it. To deal with this case, we use a technique
1.8277 +// (due to Printezis) to encode such uninitialized block sizes in the
1.8278 +// bit map. Since the bit map uses a bit per every HeapWord, but the
1.8279 +// CMS generation has a minimum object size of 3 HeapWords, it follows
1.8280 +// that "normal marks" won't be adjacent in the bit map (there will
1.8281 +// always be at least two 0 bits between successive 1 bits). We make use
1.8282 +// of these "unused" bits to represent uninitialized blocks -- the bit
1.8283 +// corresponding to the start of the uninitialized object and the next
1.8284 +// bit are both set. Finally, a 1 bit marks the end of the object that
1.8285 +// started with the two consecutive 1 bits to indicate its potentially
1.8286 +// uninitialized state.
1.8287 +
1.8288 +size_t SweepClosure::do_blk_careful(HeapWord* addr) {
1.8289 + FreeChunk* fc = (FreeChunk*)addr;
1.8290 + size_t res;
1.8291 +
1.8292 + // Check if we are done sweeping. Below we check "addr >= _limit" rather
1.8293 + // than "addr == _limit" because although _limit was a block boundary when
1.8294 + // we started the sweep, it may no longer be one because heap expansion
1.8295 + // may have caused us to coalesce the block ending at the address _limit
1.8296 + // with a newly expanded chunk (this happens when _limit was set to the
1.8297 + // previous _end of the space), so we may have stepped past _limit:
1.8298 + // see the following Zeno-like trail of CRs 6977970, 7008136, 7042740.
1.8299 + if (addr >= _limit) { // we have swept up to or past the limit: finish up
1.8300 + assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
1.8301 + "sweep _limit out of bounds");
1.8302 + assert(addr < _sp->end(), "addr out of bounds");
1.8303 + // Flush any free range we might be holding as a single
1.8304 + // coalesced chunk to the appropriate free list.
1.8305 + if (inFreeRange()) {
1.8306 + assert(freeFinger() >= _sp->bottom() && freeFinger() < _limit,
1.8307 + err_msg("freeFinger() " PTR_FORMAT" is out-of-bounds", freeFinger()));
1.8308 + flush_cur_free_chunk(freeFinger(),
1.8309 + pointer_delta(addr, freeFinger()));
1.8310 + if (CMSTraceSweeper) {
1.8311 + gclog_or_tty->print("Sweep: last chunk: ");
1.8312 + gclog_or_tty->print("put_free_blk 0x%x ("SIZE_FORMAT") "
1.8313 + "[coalesced:"SIZE_FORMAT"]\n",
1.8314 + freeFinger(), pointer_delta(addr, freeFinger()),
1.8315 + lastFreeRangeCoalesced());
1.8316 + }
1.8317 + }
1.8318 +
1.8319 + // help the iterator loop finish
1.8320 + return pointer_delta(_sp->end(), addr);
1.8321 + }
1.8322 +
1.8323 + assert(addr < _limit, "sweep invariant");
1.8324 + // check if we should yield
1.8325 + do_yield_check(addr);
1.8326 + if (fc->is_free()) {
1.8327 + // Chunk that is already free
1.8328 + res = fc->size();
1.8329 + do_already_free_chunk(fc);
1.8330 + debug_only(_sp->verifyFreeLists());
1.8331 + // If we flush the chunk at hand in lookahead_and_flush()
1.8332 + // and it's coalesced with a preceding chunk, then the
1.8333 + // process of "mangling" the payload of the coalesced block
1.8334 + // will cause erasure of the size information from the
1.8335 + // (erstwhile) header of all the coalesced blocks but the
1.8336 + // first, so the first disjunct in the assert will not hold
1.8337 + // in that specific case (in which case the second disjunct
1.8338 + // will hold).
1.8339 + assert(res == fc->size() || ((HeapWord*)fc) + res >= _limit,
1.8340 + "Otherwise the size info doesn't change at this step");
1.8341 + NOT_PRODUCT(
1.8342 + _numObjectsAlreadyFree++;
1.8343 + _numWordsAlreadyFree += res;
1.8344 + )
1.8345 + NOT_PRODUCT(_last_fc = fc;)
1.8346 + } else if (!_bitMap->isMarked(addr)) {
1.8347 + // Chunk is fresh garbage
1.8348 + res = do_garbage_chunk(fc);
1.8349 + debug_only(_sp->verifyFreeLists());
1.8350 + NOT_PRODUCT(
1.8351 + _numObjectsFreed++;
1.8352 + _numWordsFreed += res;
1.8353 + )
1.8354 + } else {
1.8355 + // Chunk that is alive.
1.8356 + res = do_live_chunk(fc);
1.8357 + debug_only(_sp->verifyFreeLists());
1.8358 + NOT_PRODUCT(
1.8359 + _numObjectsLive++;
1.8360 + _numWordsLive += res;
1.8361 + )
1.8362 + }
1.8363 + return res;
1.8364 +}
1.8365 +
1.8366 +// For the smart allocation, record following
1.8367 +// split deaths - a free chunk is removed from its free list because
1.8368 +// it is being split into two or more chunks.
1.8369 +// split birth - a free chunk is being added to its free list because
1.8370 +// a larger free chunk has been split and resulted in this free chunk.
1.8371 +// coal death - a free chunk is being removed from its free list because
1.8372 +// it is being coalesced into a large free chunk.
1.8373 +// coal birth - a free chunk is being added to its free list because
1.8374 +// it was created when two or more free chunks where coalesced into
1.8375 +// this free chunk.
1.8376 +//
1.8377 +// These statistics are used to determine the desired number of free
1.8378 +// chunks of a given size. The desired number is chosen to be relative
1.8379 +// to the end of a CMS sweep. The desired number at the end of a sweep
1.8380 +// is the
1.8381 +// count-at-end-of-previous-sweep (an amount that was enough)
1.8382 +// - count-at-beginning-of-current-sweep (the excess)
1.8383 +// + split-births (gains in this size during interval)
1.8384 +// - split-deaths (demands on this size during interval)
1.8385 +// where the interval is from the end of one sweep to the end of the
1.8386 +// next.
1.8387 +//
1.8388 +// When sweeping the sweeper maintains an accumulated chunk which is
1.8389 +// the chunk that is made up of chunks that have been coalesced. That
1.8390 +// will be termed the left-hand chunk. A new chunk of garbage that
1.8391 +// is being considered for coalescing will be referred to as the
1.8392 +// right-hand chunk.
1.8393 +//
1.8394 +// When making a decision on whether to coalesce a right-hand chunk with
1.8395 +// the current left-hand chunk, the current count vs. the desired count
1.8396 +// of the left-hand chunk is considered. Also if the right-hand chunk
1.8397 +// is near the large chunk at the end of the heap (see
1.8398 +// ConcurrentMarkSweepGeneration::isNearLargestChunk()), then the
1.8399 +// left-hand chunk is coalesced.
1.8400 +//
1.8401 +// When making a decision about whether to split a chunk, the desired count
1.8402 +// vs. the current count of the candidate to be split is also considered.
1.8403 +// If the candidate is underpopulated (currently fewer chunks than desired)
1.8404 +// a chunk of an overpopulated (currently more chunks than desired) size may
1.8405 +// be chosen. The "hint" associated with a free list, if non-null, points
1.8406 +// to a free list which may be overpopulated.
1.8407 +//
1.8408 +
1.8409 +void SweepClosure::do_already_free_chunk(FreeChunk* fc) {
1.8410 + const size_t size = fc->size();
1.8411 + // Chunks that cannot be coalesced are not in the
1.8412 + // free lists.
1.8413 + if (CMSTestInFreeList && !fc->cantCoalesce()) {
1.8414 + assert(_sp->verify_chunk_in_free_list(fc),
1.8415 + "free chunk should be in free lists");
1.8416 + }
1.8417 + // a chunk that is already free, should not have been
1.8418 + // marked in the bit map
1.8419 + HeapWord* const addr = (HeapWord*) fc;
1.8420 + assert(!_bitMap->isMarked(addr), "free chunk should be unmarked");
1.8421 + // Verify that the bit map has no bits marked between
1.8422 + // addr and purported end of this block.
1.8423 + _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
1.8424 +
1.8425 + // Some chunks cannot be coalesced under any circumstances.
1.8426 + // See the definition of cantCoalesce().
1.8427 + if (!fc->cantCoalesce()) {
1.8428 + // This chunk can potentially be coalesced.
1.8429 + if (_sp->adaptive_freelists()) {
1.8430 + // All the work is done in
1.8431 + do_post_free_or_garbage_chunk(fc, size);
1.8432 + } else { // Not adaptive free lists
1.8433 + // this is a free chunk that can potentially be coalesced by the sweeper;
1.8434 + if (!inFreeRange()) {
1.8435 + // if the next chunk is a free block that can't be coalesced
1.8436 + // it doesn't make sense to remove this chunk from the free lists
1.8437 + FreeChunk* nextChunk = (FreeChunk*)(addr + size);
1.8438 + assert((HeapWord*)nextChunk <= _sp->end(), "Chunk size out of bounds?");
1.8439 + if ((HeapWord*)nextChunk < _sp->end() && // There is another free chunk to the right ...
1.8440 + nextChunk->is_free() && // ... which is free...
1.8441 + nextChunk->cantCoalesce()) { // ... but can't be coalesced
1.8442 + // nothing to do
1.8443 + } else {
1.8444 + // Potentially the start of a new free range:
1.8445 + // Don't eagerly remove it from the free lists.
1.8446 + // No need to remove it if it will just be put
1.8447 + // back again. (Also from a pragmatic point of view
1.8448 + // if it is a free block in a region that is beyond
1.8449 + // any allocated blocks, an assertion will fail)
1.8450 + // Remember the start of a free run.
1.8451 + initialize_free_range(addr, true);
1.8452 + // end - can coalesce with next chunk
1.8453 + }
1.8454 + } else {
1.8455 + // the midst of a free range, we are coalescing
1.8456 + print_free_block_coalesced(fc);
1.8457 + if (CMSTraceSweeper) {
1.8458 + gclog_or_tty->print(" -- pick up free block 0x%x (%d)\n", fc, size);
1.8459 + }
1.8460 + // remove it from the free lists
1.8461 + _sp->removeFreeChunkFromFreeLists(fc);
1.8462 + set_lastFreeRangeCoalesced(true);
1.8463 + // If the chunk is being coalesced and the current free range is
1.8464 + // in the free lists, remove the current free range so that it
1.8465 + // will be returned to the free lists in its entirety - all
1.8466 + // the coalesced pieces included.
1.8467 + if (freeRangeInFreeLists()) {
1.8468 + FreeChunk* ffc = (FreeChunk*) freeFinger();
1.8469 + assert(ffc->size() == pointer_delta(addr, freeFinger()),
1.8470 + "Size of free range is inconsistent with chunk size.");
1.8471 + if (CMSTestInFreeList) {
1.8472 + assert(_sp->verify_chunk_in_free_list(ffc),
1.8473 + "free range is not in free lists");
1.8474 + }
1.8475 + _sp->removeFreeChunkFromFreeLists(ffc);
1.8476 + set_freeRangeInFreeLists(false);
1.8477 + }
1.8478 + }
1.8479 + }
1.8480 + // Note that if the chunk is not coalescable (the else arm
1.8481 + // below), we unconditionally flush, without needing to do
1.8482 + // a "lookahead," as we do below.
1.8483 + if (inFreeRange()) lookahead_and_flush(fc, size);
1.8484 + } else {
1.8485 + // Code path common to both original and adaptive free lists.
1.8486 +
1.8487 + // cant coalesce with previous block; this should be treated
1.8488 + // as the end of a free run if any
1.8489 + if (inFreeRange()) {
1.8490 + // we kicked some butt; time to pick up the garbage
1.8491 + assert(freeFinger() < addr, "freeFinger points too high");
1.8492 + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger()));
1.8493 + }
1.8494 + // else, nothing to do, just continue
1.8495 + }
1.8496 +}
1.8497 +
1.8498 +size_t SweepClosure::do_garbage_chunk(FreeChunk* fc) {
1.8499 + // This is a chunk of garbage. It is not in any free list.
1.8500 + // Add it to a free list or let it possibly be coalesced into
1.8501 + // a larger chunk.
1.8502 + HeapWord* const addr = (HeapWord*) fc;
1.8503 + const size_t size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
1.8504 +
1.8505 + if (_sp->adaptive_freelists()) {
1.8506 + // Verify that the bit map has no bits marked between
1.8507 + // addr and purported end of just dead object.
1.8508 + _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
1.8509 +
1.8510 + do_post_free_or_garbage_chunk(fc, size);
1.8511 + } else {
1.8512 + if (!inFreeRange()) {
1.8513 + // start of a new free range
1.8514 + assert(size > 0, "A free range should have a size");
1.8515 + initialize_free_range(addr, false);
1.8516 + } else {
1.8517 + // this will be swept up when we hit the end of the
1.8518 + // free range
1.8519 + if (CMSTraceSweeper) {
1.8520 + gclog_or_tty->print(" -- pick up garbage 0x%x (%d) \n", fc, size);
1.8521 + }
1.8522 + // If the chunk is being coalesced and the current free range is
1.8523 + // in the free lists, remove the current free range so that it
1.8524 + // will be returned to the free lists in its entirety - all
1.8525 + // the coalesced pieces included.
1.8526 + if (freeRangeInFreeLists()) {
1.8527 + FreeChunk* ffc = (FreeChunk*)freeFinger();
1.8528 + assert(ffc->size() == pointer_delta(addr, freeFinger()),
1.8529 + "Size of free range is inconsistent with chunk size.");
1.8530 + if (CMSTestInFreeList) {
1.8531 + assert(_sp->verify_chunk_in_free_list(ffc),
1.8532 + "free range is not in free lists");
1.8533 + }
1.8534 + _sp->removeFreeChunkFromFreeLists(ffc);
1.8535 + set_freeRangeInFreeLists(false);
1.8536 + }
1.8537 + set_lastFreeRangeCoalesced(true);
1.8538 + }
1.8539 + // this will be swept up when we hit the end of the free range
1.8540 +
1.8541 + // Verify that the bit map has no bits marked between
1.8542 + // addr and purported end of just dead object.
1.8543 + _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
1.8544 + }
1.8545 + assert(_limit >= addr + size,
1.8546 + "A freshly garbage chunk can't possibly straddle over _limit");
1.8547 + if (inFreeRange()) lookahead_and_flush(fc, size);
1.8548 + return size;
1.8549 +}
1.8550 +
1.8551 +size_t SweepClosure::do_live_chunk(FreeChunk* fc) {
1.8552 + HeapWord* addr = (HeapWord*) fc;
1.8553 + // The sweeper has just found a live object. Return any accumulated
1.8554 + // left hand chunk to the free lists.
1.8555 + if (inFreeRange()) {
1.8556 + assert(freeFinger() < addr, "freeFinger points too high");
1.8557 + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger()));
1.8558 + }
1.8559 +
1.8560 + // This object is live: we'd normally expect this to be
1.8561 + // an oop, and like to assert the following:
1.8562 + // assert(oop(addr)->is_oop(), "live block should be an oop");
1.8563 + // However, as we commented above, this may be an object whose
1.8564 + // header hasn't yet been initialized.
1.8565 + size_t size;
1.8566 + assert(_bitMap->isMarked(addr), "Tautology for this control point");
1.8567 + if (_bitMap->isMarked(addr + 1)) {
1.8568 + // Determine the size from the bit map, rather than trying to
1.8569 + // compute it from the object header.
1.8570 + HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
1.8571 + size = pointer_delta(nextOneAddr + 1, addr);
1.8572 + assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
1.8573 + "alignment problem");
1.8574 +
1.8575 +#ifdef ASSERT
1.8576 + if (oop(addr)->klass_or_null() != NULL) {
1.8577 + // Ignore mark word because we are running concurrent with mutators
1.8578 + assert(oop(addr)->is_oop(true), "live block should be an oop");
1.8579 + assert(size ==
1.8580 + CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()),
1.8581 + "P-mark and computed size do not agree");
1.8582 + }
1.8583 +#endif
1.8584 +
1.8585 + } else {
1.8586 + // This should be an initialized object that's alive.
1.8587 + assert(oop(addr)->klass_or_null() != NULL,
1.8588 + "Should be an initialized object");
1.8589 + // Ignore mark word because we are running concurrent with mutators
1.8590 + assert(oop(addr)->is_oop(true), "live block should be an oop");
1.8591 + // Verify that the bit map has no bits marked between
1.8592 + // addr and purported end of this block.
1.8593 + size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
1.8594 + assert(size >= 3, "Necessary for Printezis marks to work");
1.8595 + assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
1.8596 + DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
1.8597 + }
1.8598 + return size;
1.8599 +}
1.8600 +
1.8601 +void SweepClosure::do_post_free_or_garbage_chunk(FreeChunk* fc,
1.8602 + size_t chunkSize) {
1.8603 + // do_post_free_or_garbage_chunk() should only be called in the case
1.8604 + // of the adaptive free list allocator.
1.8605 + const bool fcInFreeLists = fc->is_free();
1.8606 + assert(_sp->adaptive_freelists(), "Should only be used in this case.");
1.8607 + assert((HeapWord*)fc <= _limit, "sweep invariant");
1.8608 + if (CMSTestInFreeList && fcInFreeLists) {
1.8609 + assert(_sp->verify_chunk_in_free_list(fc), "free chunk is not in free lists");
1.8610 + }
1.8611 +
1.8612 + if (CMSTraceSweeper) {
1.8613 + gclog_or_tty->print_cr(" -- pick up another chunk at 0x%x (%d)", fc, chunkSize);
1.8614 + }
1.8615 +
1.8616 + HeapWord* const fc_addr = (HeapWord*) fc;
1.8617 +
1.8618 + bool coalesce;
1.8619 + const size_t left = pointer_delta(fc_addr, freeFinger());
1.8620 + const size_t right = chunkSize;
1.8621 + switch (FLSCoalescePolicy) {
1.8622 + // numeric value forms a coalition aggressiveness metric
1.8623 + case 0: { // never coalesce
1.8624 + coalesce = false;
1.8625 + break;
1.8626 + }
1.8627 + case 1: { // coalesce if left & right chunks on overpopulated lists
1.8628 + coalesce = _sp->coalOverPopulated(left) &&
1.8629 + _sp->coalOverPopulated(right);
1.8630 + break;
1.8631 + }
1.8632 + case 2: { // coalesce if left chunk on overpopulated list (default)
1.8633 + coalesce = _sp->coalOverPopulated(left);
1.8634 + break;
1.8635 + }
1.8636 + case 3: { // coalesce if left OR right chunk on overpopulated list
1.8637 + coalesce = _sp->coalOverPopulated(left) ||
1.8638 + _sp->coalOverPopulated(right);
1.8639 + break;
1.8640 + }
1.8641 + case 4: { // always coalesce
1.8642 + coalesce = true;
1.8643 + break;
1.8644 + }
1.8645 + default:
1.8646 + ShouldNotReachHere();
1.8647 + }
1.8648 +
1.8649 + // Should the current free range be coalesced?
1.8650 + // If the chunk is in a free range and either we decided to coalesce above
1.8651 + // or the chunk is near the large block at the end of the heap
1.8652 + // (isNearLargestChunk() returns true), then coalesce this chunk.
1.8653 + const bool doCoalesce = inFreeRange()
1.8654 + && (coalesce || _g->isNearLargestChunk(fc_addr));
1.8655 + if (doCoalesce) {
1.8656 + // Coalesce the current free range on the left with the new
1.8657 + // chunk on the right. If either is on a free list,
1.8658 + // it must be removed from the list and stashed in the closure.
1.8659 + if (freeRangeInFreeLists()) {
1.8660 + FreeChunk* const ffc = (FreeChunk*)freeFinger();
1.8661 + assert(ffc->size() == pointer_delta(fc_addr, freeFinger()),
1.8662 + "Size of free range is inconsistent with chunk size.");
1.8663 + if (CMSTestInFreeList) {
1.8664 + assert(_sp->verify_chunk_in_free_list(ffc),
1.8665 + "Chunk is not in free lists");
1.8666 + }
1.8667 + _sp->coalDeath(ffc->size());
1.8668 + _sp->removeFreeChunkFromFreeLists(ffc);
1.8669 + set_freeRangeInFreeLists(false);
1.8670 + }
1.8671 + if (fcInFreeLists) {
1.8672 + _sp->coalDeath(chunkSize);
1.8673 + assert(fc->size() == chunkSize,
1.8674 + "The chunk has the wrong size or is not in the free lists");
1.8675 + _sp->removeFreeChunkFromFreeLists(fc);
1.8676 + }
1.8677 + set_lastFreeRangeCoalesced(true);
1.8678 + print_free_block_coalesced(fc);
1.8679 + } else { // not in a free range and/or should not coalesce
1.8680 + // Return the current free range and start a new one.
1.8681 + if (inFreeRange()) {
1.8682 + // In a free range but cannot coalesce with the right hand chunk.
1.8683 + // Put the current free range into the free lists.
1.8684 + flush_cur_free_chunk(freeFinger(),
1.8685 + pointer_delta(fc_addr, freeFinger()));
1.8686 + }
1.8687 + // Set up for new free range. Pass along whether the right hand
1.8688 + // chunk is in the free lists.
1.8689 + initialize_free_range((HeapWord*)fc, fcInFreeLists);
1.8690 + }
1.8691 +}
1.8692 +
1.8693 +// Lookahead flush:
1.8694 +// If we are tracking a free range, and this is the last chunk that
1.8695 +// we'll look at because its end crosses past _limit, we'll preemptively
1.8696 +// flush it along with any free range we may be holding on to. Note that
1.8697 +// this can be the case only for an already free or freshly garbage
1.8698 +// chunk. If this block is an object, it can never straddle
1.8699 +// over _limit. The "straddling" occurs when _limit is set at
1.8700 +// the previous end of the space when this cycle started, and
1.8701 +// a subsequent heap expansion caused the previously co-terminal
1.8702 +// free block to be coalesced with the newly expanded portion,
1.8703 +// thus rendering _limit a non-block-boundary making it dangerous
1.8704 +// for the sweeper to step over and examine.
1.8705 +void SweepClosure::lookahead_and_flush(FreeChunk* fc, size_t chunk_size) {
1.8706 + assert(inFreeRange(), "Should only be called if currently in a free range.");
1.8707 + HeapWord* const eob = ((HeapWord*)fc) + chunk_size;
1.8708 + assert(_sp->used_region().contains(eob - 1),
1.8709 + err_msg("eob = " PTR_FORMAT " eob-1 = " PTR_FORMAT " _limit = " PTR_FORMAT
1.8710 + " out of bounds wrt _sp = [" PTR_FORMAT "," PTR_FORMAT ")"
1.8711 + " when examining fc = " PTR_FORMAT "(" SIZE_FORMAT ")",
1.8712 + eob, eob-1, _limit, _sp->bottom(), _sp->end(), fc, chunk_size));
1.8713 + if (eob >= _limit) {
1.8714 + assert(eob == _limit || fc->is_free(), "Only a free chunk should allow us to cross over the limit");
1.8715 + if (CMSTraceSweeper) {
1.8716 + gclog_or_tty->print_cr("_limit " PTR_FORMAT " reached or crossed by block "
1.8717 + "[" PTR_FORMAT "," PTR_FORMAT ") in space "
1.8718 + "[" PTR_FORMAT "," PTR_FORMAT ")",
1.8719 + _limit, fc, eob, _sp->bottom(), _sp->end());
1.8720 + }
1.8721 + // Return the storage we are tracking back into the free lists.
1.8722 + if (CMSTraceSweeper) {
1.8723 + gclog_or_tty->print_cr("Flushing ... ");
1.8724 + }
1.8725 + assert(freeFinger() < eob, "Error");
1.8726 + flush_cur_free_chunk( freeFinger(), pointer_delta(eob, freeFinger()));
1.8727 + }
1.8728 +}
1.8729 +
1.8730 +void SweepClosure::flush_cur_free_chunk(HeapWord* chunk, size_t size) {
1.8731 + assert(inFreeRange(), "Should only be called if currently in a free range.");
1.8732 + assert(size > 0,
1.8733 + "A zero sized chunk cannot be added to the free lists.");
1.8734 + if (!freeRangeInFreeLists()) {
1.8735 + if (CMSTestInFreeList) {
1.8736 + FreeChunk* fc = (FreeChunk*) chunk;
1.8737 + fc->set_size(size);
1.8738 + assert(!_sp->verify_chunk_in_free_list(fc),
1.8739 + "chunk should not be in free lists yet");
1.8740 + }
1.8741 + if (CMSTraceSweeper) {
1.8742 + gclog_or_tty->print_cr(" -- add free block 0x%x (%d) to free lists",
1.8743 + chunk, size);
1.8744 + }
1.8745 + // A new free range is going to be starting. The current
1.8746 + // free range has not been added to the free lists yet or
1.8747 + // was removed so add it back.
1.8748 + // If the current free range was coalesced, then the death
1.8749 + // of the free range was recorded. Record a birth now.
1.8750 + if (lastFreeRangeCoalesced()) {
1.8751 + _sp->coalBirth(size);
1.8752 + }
1.8753 + _sp->addChunkAndRepairOffsetTable(chunk, size,
1.8754 + lastFreeRangeCoalesced());
1.8755 + } else if (CMSTraceSweeper) {
1.8756 + gclog_or_tty->print_cr("Already in free list: nothing to flush");
1.8757 + }
1.8758 + set_inFreeRange(false);
1.8759 + set_freeRangeInFreeLists(false);
1.8760 +}
1.8761 +
1.8762 +// We take a break if we've been at this for a while,
1.8763 +// so as to avoid monopolizing the locks involved.
1.8764 +void SweepClosure::do_yield_work(HeapWord* addr) {
1.8765 + // Return current free chunk being used for coalescing (if any)
1.8766 + // to the appropriate freelist. After yielding, the next
1.8767 + // free block encountered will start a coalescing range of
1.8768 + // free blocks. If the next free block is adjacent to the
1.8769 + // chunk just flushed, they will need to wait for the next
1.8770 + // sweep to be coalesced.
1.8771 + if (inFreeRange()) {
1.8772 + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger()));
1.8773 + }
1.8774 +
1.8775 + // First give up the locks, then yield, then re-lock.
1.8776 + // We should probably use a constructor/destructor idiom to
1.8777 + // do this unlock/lock or modify the MutexUnlocker class to
1.8778 + // serve our purpose. XXX
1.8779 + assert_lock_strong(_bitMap->lock());
1.8780 + assert_lock_strong(_freelistLock);
1.8781 + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1.8782 + "CMS thread should hold CMS token");
1.8783 + _bitMap->lock()->unlock();
1.8784 + _freelistLock->unlock();
1.8785 + ConcurrentMarkSweepThread::desynchronize(true);
1.8786 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.8787 + _collector->stopTimer();
1.8788 + GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
1.8789 + if (PrintCMSStatistics != 0) {
1.8790 + _collector->incrementYields();
1.8791 + }
1.8792 + _collector->icms_wait();
1.8793 +
1.8794 + // See the comment in coordinator_yield()
1.8795 + for (unsigned i = 0; i < CMSYieldSleepCount &&
1.8796 + ConcurrentMarkSweepThread::should_yield() &&
1.8797 + !CMSCollector::foregroundGCIsActive(); ++i) {
1.8798 + os::sleep(Thread::current(), 1, false);
1.8799 + ConcurrentMarkSweepThread::acknowledge_yield_request();
1.8800 + }
1.8801 +
1.8802 + ConcurrentMarkSweepThread::synchronize(true);
1.8803 + _freelistLock->lock();
1.8804 + _bitMap->lock()->lock_without_safepoint_check();
1.8805 + _collector->startTimer();
1.8806 +}
1.8807 +
1.8808 +#ifndef PRODUCT
1.8809 +// This is actually very useful in a product build if it can
1.8810 +// be called from the debugger. Compile it into the product
1.8811 +// as needed.
1.8812 +bool debug_verify_chunk_in_free_list(FreeChunk* fc) {
1.8813 + return debug_cms_space->verify_chunk_in_free_list(fc);
1.8814 +}
1.8815 +#endif
1.8816 +
1.8817 +void SweepClosure::print_free_block_coalesced(FreeChunk* fc) const {
1.8818 + if (CMSTraceSweeper) {
1.8819 + gclog_or_tty->print_cr("Sweep:coal_free_blk " PTR_FORMAT " (" SIZE_FORMAT ")",
1.8820 + fc, fc->size());
1.8821 + }
1.8822 +}
1.8823 +
1.8824 +// CMSIsAliveClosure
1.8825 +bool CMSIsAliveClosure::do_object_b(oop obj) {
1.8826 + HeapWord* addr = (HeapWord*)obj;
1.8827 + return addr != NULL &&
1.8828 + (!_span.contains(addr) || _bit_map->isMarked(addr));
1.8829 +}
1.8830 +
1.8831 +
1.8832 +CMSKeepAliveClosure::CMSKeepAliveClosure( CMSCollector* collector,
1.8833 + MemRegion span,
1.8834 + CMSBitMap* bit_map, CMSMarkStack* mark_stack,
1.8835 + bool cpc):
1.8836 + _collector(collector),
1.8837 + _span(span),
1.8838 + _bit_map(bit_map),
1.8839 + _mark_stack(mark_stack),
1.8840 + _concurrent_precleaning(cpc) {
1.8841 + assert(!_span.is_empty(), "Empty span could spell trouble");
1.8842 +}
1.8843 +
1.8844 +
1.8845 +// CMSKeepAliveClosure: the serial version
1.8846 +void CMSKeepAliveClosure::do_oop(oop obj) {
1.8847 + HeapWord* addr = (HeapWord*)obj;
1.8848 + if (_span.contains(addr) &&
1.8849 + !_bit_map->isMarked(addr)) {
1.8850 + _bit_map->mark(addr);
1.8851 + bool simulate_overflow = false;
1.8852 + NOT_PRODUCT(
1.8853 + if (CMSMarkStackOverflowALot &&
1.8854 + _collector->simulate_overflow()) {
1.8855 + // simulate a stack overflow
1.8856 + simulate_overflow = true;
1.8857 + }
1.8858 + )
1.8859 + if (simulate_overflow || !_mark_stack->push(obj)) {
1.8860 + if (_concurrent_precleaning) {
1.8861 + // We dirty the overflown object and let the remark
1.8862 + // phase deal with it.
1.8863 + assert(_collector->overflow_list_is_empty(), "Error");
1.8864 + // In the case of object arrays, we need to dirty all of
1.8865 + // the cards that the object spans. No locking or atomics
1.8866 + // are needed since no one else can be mutating the mod union
1.8867 + // table.
1.8868 + if (obj->is_objArray()) {
1.8869 + size_t sz = obj->size();
1.8870 + HeapWord* end_card_addr =
1.8871 + (HeapWord*)round_to((intptr_t)(addr+sz), CardTableModRefBS::card_size);
1.8872 + MemRegion redirty_range = MemRegion(addr, end_card_addr);
1.8873 + assert(!redirty_range.is_empty(), "Arithmetical tautology");
1.8874 + _collector->_modUnionTable.mark_range(redirty_range);
1.8875 + } else {
1.8876 + _collector->_modUnionTable.mark(addr);
1.8877 + }
1.8878 + _collector->_ser_kac_preclean_ovflw++;
1.8879 + } else {
1.8880 + _collector->push_on_overflow_list(obj);
1.8881 + _collector->_ser_kac_ovflw++;
1.8882 + }
1.8883 + }
1.8884 + }
1.8885 +}
1.8886 +
1.8887 +void CMSKeepAliveClosure::do_oop(oop* p) { CMSKeepAliveClosure::do_oop_work(p); }
1.8888 +void CMSKeepAliveClosure::do_oop(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); }
1.8889 +
1.8890 +// CMSParKeepAliveClosure: a parallel version of the above.
1.8891 +// The work queues are private to each closure (thread),
1.8892 +// but (may be) available for stealing by other threads.
1.8893 +void CMSParKeepAliveClosure::do_oop(oop obj) {
1.8894 + HeapWord* addr = (HeapWord*)obj;
1.8895 + if (_span.contains(addr) &&
1.8896 + !_bit_map->isMarked(addr)) {
1.8897 + // In general, during recursive tracing, several threads
1.8898 + // may be concurrently getting here; the first one to
1.8899 + // "tag" it, claims it.
1.8900 + if (_bit_map->par_mark(addr)) {
1.8901 + bool res = _work_queue->push(obj);
1.8902 + assert(res, "Low water mark should be much less than capacity");
1.8903 + // Do a recursive trim in the hope that this will keep
1.8904 + // stack usage lower, but leave some oops for potential stealers
1.8905 + trim_queue(_low_water_mark);
1.8906 + } // Else, another thread got there first
1.8907 + }
1.8908 +}
1.8909 +
1.8910 +void CMSParKeepAliveClosure::do_oop(oop* p) { CMSParKeepAliveClosure::do_oop_work(p); }
1.8911 +void CMSParKeepAliveClosure::do_oop(narrowOop* p) { CMSParKeepAliveClosure::do_oop_work(p); }
1.8912 +
1.8913 +void CMSParKeepAliveClosure::trim_queue(uint max) {
1.8914 + while (_work_queue->size() > max) {
1.8915 + oop new_oop;
1.8916 + if (_work_queue->pop_local(new_oop)) {
1.8917 + assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
1.8918 + assert(_bit_map->isMarked((HeapWord*)new_oop),
1.8919 + "no white objects on this stack!");
1.8920 + assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop");
1.8921 + // iterate over the oops in this oop, marking and pushing
1.8922 + // the ones in CMS heap (i.e. in _span).
1.8923 + new_oop->oop_iterate(&_mark_and_push);
1.8924 + }
1.8925 + }
1.8926 +}
1.8927 +
1.8928 +CMSInnerParMarkAndPushClosure::CMSInnerParMarkAndPushClosure(
1.8929 + CMSCollector* collector,
1.8930 + MemRegion span, CMSBitMap* bit_map,
1.8931 + OopTaskQueue* work_queue):
1.8932 + _collector(collector),
1.8933 + _span(span),
1.8934 + _bit_map(bit_map),
1.8935 + _work_queue(work_queue) { }
1.8936 +
1.8937 +void CMSInnerParMarkAndPushClosure::do_oop(oop obj) {
1.8938 + HeapWord* addr = (HeapWord*)obj;
1.8939 + if (_span.contains(addr) &&
1.8940 + !_bit_map->isMarked(addr)) {
1.8941 + if (_bit_map->par_mark(addr)) {
1.8942 + bool simulate_overflow = false;
1.8943 + NOT_PRODUCT(
1.8944 + if (CMSMarkStackOverflowALot &&
1.8945 + _collector->par_simulate_overflow()) {
1.8946 + // simulate a stack overflow
1.8947 + simulate_overflow = true;
1.8948 + }
1.8949 + )
1.8950 + if (simulate_overflow || !_work_queue->push(obj)) {
1.8951 + _collector->par_push_on_overflow_list(obj);
1.8952 + _collector->_par_kac_ovflw++;
1.8953 + }
1.8954 + } // Else another thread got there already
1.8955 + }
1.8956 +}
1.8957 +
1.8958 +void CMSInnerParMarkAndPushClosure::do_oop(oop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); }
1.8959 +void CMSInnerParMarkAndPushClosure::do_oop(narrowOop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); }
1.8960 +
1.8961 +//////////////////////////////////////////////////////////////////
1.8962 +// CMSExpansionCause /////////////////////////////
1.8963 +//////////////////////////////////////////////////////////////////
1.8964 +const char* CMSExpansionCause::to_string(CMSExpansionCause::Cause cause) {
1.8965 + switch (cause) {
1.8966 + case _no_expansion:
1.8967 + return "No expansion";
1.8968 + case _satisfy_free_ratio:
1.8969 + return "Free ratio";
1.8970 + case _satisfy_promotion:
1.8971 + return "Satisfy promotion";
1.8972 + case _satisfy_allocation:
1.8973 + return "allocation";
1.8974 + case _allocate_par_lab:
1.8975 + return "Par LAB";
1.8976 + case _allocate_par_spooling_space:
1.8977 + return "Par Spooling Space";
1.8978 + case _adaptive_size_policy:
1.8979 + return "Ergonomics";
1.8980 + default:
1.8981 + return "unknown";
1.8982 + }
1.8983 +}
1.8984 +
1.8985 +void CMSDrainMarkingStackClosure::do_void() {
1.8986 + // the max number to take from overflow list at a time
1.8987 + const size_t num = _mark_stack->capacity()/4;
1.8988 + assert(!_concurrent_precleaning || _collector->overflow_list_is_empty(),
1.8989 + "Overflow list should be NULL during concurrent phases");
1.8990 + while (!_mark_stack->isEmpty() ||
1.8991 + // if stack is empty, check the overflow list
1.8992 + _collector->take_from_overflow_list(num, _mark_stack)) {
1.8993 + oop obj = _mark_stack->pop();
1.8994 + HeapWord* addr = (HeapWord*)obj;
1.8995 + assert(_span.contains(addr), "Should be within span");
1.8996 + assert(_bit_map->isMarked(addr), "Should be marked");
1.8997 + assert(obj->is_oop(), "Should be an oop");
1.8998 + obj->oop_iterate(_keep_alive);
1.8999 + }
1.9000 +}
1.9001 +
1.9002 +void CMSParDrainMarkingStackClosure::do_void() {
1.9003 + // drain queue
1.9004 + trim_queue(0);
1.9005 +}
1.9006 +
1.9007 +// Trim our work_queue so its length is below max at return
1.9008 +void CMSParDrainMarkingStackClosure::trim_queue(uint max) {
1.9009 + while (_work_queue->size() > max) {
1.9010 + oop new_oop;
1.9011 + if (_work_queue->pop_local(new_oop)) {
1.9012 + assert(new_oop->is_oop(), "Expected an oop");
1.9013 + assert(_bit_map->isMarked((HeapWord*)new_oop),
1.9014 + "no white objects on this stack!");
1.9015 + assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop");
1.9016 + // iterate over the oops in this oop, marking and pushing
1.9017 + // the ones in CMS heap (i.e. in _span).
1.9018 + new_oop->oop_iterate(&_mark_and_push);
1.9019 + }
1.9020 + }
1.9021 +}
1.9022 +
1.9023 +////////////////////////////////////////////////////////////////////
1.9024 +// Support for Marking Stack Overflow list handling and related code
1.9025 +////////////////////////////////////////////////////////////////////
1.9026 +// Much of the following code is similar in shape and spirit to the
1.9027 +// code used in ParNewGC. We should try and share that code
1.9028 +// as much as possible in the future.
1.9029 +
1.9030 +#ifndef PRODUCT
1.9031 +// Debugging support for CMSStackOverflowALot
1.9032 +
1.9033 +// It's OK to call this multi-threaded; the worst thing
1.9034 +// that can happen is that we'll get a bunch of closely
1.9035 +// spaced simulated oveflows, but that's OK, in fact
1.9036 +// probably good as it would exercise the overflow code
1.9037 +// under contention.
1.9038 +bool CMSCollector::simulate_overflow() {
1.9039 + if (_overflow_counter-- <= 0) { // just being defensive
1.9040 + _overflow_counter = CMSMarkStackOverflowInterval;
1.9041 + return true;
1.9042 + } else {
1.9043 + return false;
1.9044 + }
1.9045 +}
1.9046 +
1.9047 +bool CMSCollector::par_simulate_overflow() {
1.9048 + return simulate_overflow();
1.9049 +}
1.9050 +#endif
1.9051 +
1.9052 +// Single-threaded
1.9053 +bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) {
1.9054 + assert(stack->isEmpty(), "Expected precondition");
1.9055 + assert(stack->capacity() > num, "Shouldn't bite more than can chew");
1.9056 + size_t i = num;
1.9057 + oop cur = _overflow_list;
1.9058 + const markOop proto = markOopDesc::prototype();
1.9059 + NOT_PRODUCT(ssize_t n = 0;)
1.9060 + for (oop next; i > 0 && cur != NULL; cur = next, i--) {
1.9061 + next = oop(cur->mark());
1.9062 + cur->set_mark(proto); // until proven otherwise
1.9063 + assert(cur->is_oop(), "Should be an oop");
1.9064 + bool res = stack->push(cur);
1.9065 + assert(res, "Bit off more than can chew?");
1.9066 + NOT_PRODUCT(n++;)
1.9067 + }
1.9068 + _overflow_list = cur;
1.9069 +#ifndef PRODUCT
1.9070 + assert(_num_par_pushes >= n, "Too many pops?");
1.9071 + _num_par_pushes -=n;
1.9072 +#endif
1.9073 + return !stack->isEmpty();
1.9074 +}
1.9075 +
1.9076 +#define BUSY (cast_to_oop<intptr_t>(0x1aff1aff))
1.9077 +// (MT-safe) Get a prefix of at most "num" from the list.
1.9078 +// The overflow list is chained through the mark word of
1.9079 +// each object in the list. We fetch the entire list,
1.9080 +// break off a prefix of the right size and return the
1.9081 +// remainder. If other threads try to take objects from
1.9082 +// the overflow list at that time, they will wait for
1.9083 +// some time to see if data becomes available. If (and
1.9084 +// only if) another thread places one or more object(s)
1.9085 +// on the global list before we have returned the suffix
1.9086 +// to the global list, we will walk down our local list
1.9087 +// to find its end and append the global list to
1.9088 +// our suffix before returning it. This suffix walk can
1.9089 +// prove to be expensive (quadratic in the amount of traffic)
1.9090 +// when there are many objects in the overflow list and
1.9091 +// there is much producer-consumer contention on the list.
1.9092 +// *NOTE*: The overflow list manipulation code here and
1.9093 +// in ParNewGeneration:: are very similar in shape,
1.9094 +// except that in the ParNew case we use the old (from/eden)
1.9095 +// copy of the object to thread the list via its klass word.
1.9096 +// Because of the common code, if you make any changes in
1.9097 +// the code below, please check the ParNew version to see if
1.9098 +// similar changes might be needed.
1.9099 +// CR 6797058 has been filed to consolidate the common code.
1.9100 +bool CMSCollector::par_take_from_overflow_list(size_t num,
1.9101 + OopTaskQueue* work_q,
1.9102 + int no_of_gc_threads) {
1.9103 + assert(work_q->size() == 0, "First empty local work queue");
1.9104 + assert(num < work_q->max_elems(), "Can't bite more than we can chew");
1.9105 + if (_overflow_list == NULL) {
1.9106 + return false;
1.9107 + }
1.9108 + // Grab the entire list; we'll put back a suffix
1.9109 + oop prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list));
1.9110 + Thread* tid = Thread::current();
1.9111 + // Before "no_of_gc_threads" was introduced CMSOverflowSpinCount was
1.9112 + // set to ParallelGCThreads.
1.9113 + size_t CMSOverflowSpinCount = (size_t) no_of_gc_threads; // was ParallelGCThreads;
1.9114 + size_t sleep_time_millis = MAX2((size_t)1, num/100);
1.9115 + // If the list is busy, we spin for a short while,
1.9116 + // sleeping between attempts to get the list.
1.9117 + for (size_t spin = 0; prefix == BUSY && spin < CMSOverflowSpinCount; spin++) {
1.9118 + os::sleep(tid, sleep_time_millis, false);
1.9119 + if (_overflow_list == NULL) {
1.9120 + // Nothing left to take
1.9121 + return false;
1.9122 + } else if (_overflow_list != BUSY) {
1.9123 + // Try and grab the prefix
1.9124 + prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list));
1.9125 + }
1.9126 + }
1.9127 + // If the list was found to be empty, or we spun long
1.9128 + // enough, we give up and return empty-handed. If we leave
1.9129 + // the list in the BUSY state below, it must be the case that
1.9130 + // some other thread holds the overflow list and will set it
1.9131 + // to a non-BUSY state in the future.
1.9132 + if (prefix == NULL || prefix == BUSY) {
1.9133 + // Nothing to take or waited long enough
1.9134 + if (prefix == NULL) {
1.9135 + // Write back the NULL in case we overwrote it with BUSY above
1.9136 + // and it is still the same value.
1.9137 + (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY);
1.9138 + }
1.9139 + return false;
1.9140 + }
1.9141 + assert(prefix != NULL && prefix != BUSY, "Error");
1.9142 + size_t i = num;
1.9143 + oop cur = prefix;
1.9144 + // Walk down the first "num" objects, unless we reach the end.
1.9145 + for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--);
1.9146 + if (cur->mark() == NULL) {
1.9147 + // We have "num" or fewer elements in the list, so there
1.9148 + // is nothing to return to the global list.
1.9149 + // Write back the NULL in lieu of the BUSY we wrote
1.9150 + // above, if it is still the same value.
1.9151 + if (_overflow_list == BUSY) {
1.9152 + (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY);
1.9153 + }
1.9154 + } else {
1.9155 + // Chop off the suffix and rerturn it to the global list.
1.9156 + assert(cur->mark() != BUSY, "Error");
1.9157 + oop suffix_head = cur->mark(); // suffix will be put back on global list
1.9158 + cur->set_mark(NULL); // break off suffix
1.9159 + // It's possible that the list is still in the empty(busy) state
1.9160 + // we left it in a short while ago; in that case we may be
1.9161 + // able to place back the suffix without incurring the cost
1.9162 + // of a walk down the list.
1.9163 + oop observed_overflow_list = _overflow_list;
1.9164 + oop cur_overflow_list = observed_overflow_list;
1.9165 + bool attached = false;
1.9166 + while (observed_overflow_list == BUSY || observed_overflow_list == NULL) {
1.9167 + observed_overflow_list =
1.9168 + (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur_overflow_list);
1.9169 + if (cur_overflow_list == observed_overflow_list) {
1.9170 + attached = true;
1.9171 + break;
1.9172 + } else cur_overflow_list = observed_overflow_list;
1.9173 + }
1.9174 + if (!attached) {
1.9175 + // Too bad, someone else sneaked in (at least) an element; we'll need
1.9176 + // to do a splice. Find tail of suffix so we can prepend suffix to global
1.9177 + // list.
1.9178 + for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark()));
1.9179 + oop suffix_tail = cur;
1.9180 + assert(suffix_tail != NULL && suffix_tail->mark() == NULL,
1.9181 + "Tautology");
1.9182 + observed_overflow_list = _overflow_list;
1.9183 + do {
1.9184 + cur_overflow_list = observed_overflow_list;
1.9185 + if (cur_overflow_list != BUSY) {
1.9186 + // Do the splice ...
1.9187 + suffix_tail->set_mark(markOop(cur_overflow_list));
1.9188 + } else { // cur_overflow_list == BUSY
1.9189 + suffix_tail->set_mark(NULL);
1.9190 + }
1.9191 + // ... and try to place spliced list back on overflow_list ...
1.9192 + observed_overflow_list =
1.9193 + (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur_overflow_list);
1.9194 + } while (cur_overflow_list != observed_overflow_list);
1.9195 + // ... until we have succeeded in doing so.
1.9196 + }
1.9197 + }
1.9198 +
1.9199 + // Push the prefix elements on work_q
1.9200 + assert(prefix != NULL, "control point invariant");
1.9201 + const markOop proto = markOopDesc::prototype();
1.9202 + oop next;
1.9203 + NOT_PRODUCT(ssize_t n = 0;)
1.9204 + for (cur = prefix; cur != NULL; cur = next) {
1.9205 + next = oop(cur->mark());
1.9206 + cur->set_mark(proto); // until proven otherwise
1.9207 + assert(cur->is_oop(), "Should be an oop");
1.9208 + bool res = work_q->push(cur);
1.9209 + assert(res, "Bit off more than we can chew?");
1.9210 + NOT_PRODUCT(n++;)
1.9211 + }
1.9212 +#ifndef PRODUCT
1.9213 + assert(_num_par_pushes >= n, "Too many pops?");
1.9214 + Atomic::add_ptr(-(intptr_t)n, &_num_par_pushes);
1.9215 +#endif
1.9216 + return true;
1.9217 +}
1.9218 +
1.9219 +// Single-threaded
1.9220 +void CMSCollector::push_on_overflow_list(oop p) {
1.9221 + NOT_PRODUCT(_num_par_pushes++;)
1.9222 + assert(p->is_oop(), "Not an oop");
1.9223 + preserve_mark_if_necessary(p);
1.9224 + p->set_mark((markOop)_overflow_list);
1.9225 + _overflow_list = p;
1.9226 +}
1.9227 +
1.9228 +// Multi-threaded; use CAS to prepend to overflow list
1.9229 +void CMSCollector::par_push_on_overflow_list(oop p) {
1.9230 + NOT_PRODUCT(Atomic::inc_ptr(&_num_par_pushes);)
1.9231 + assert(p->is_oop(), "Not an oop");
1.9232 + par_preserve_mark_if_necessary(p);
1.9233 + oop observed_overflow_list = _overflow_list;
1.9234 + oop cur_overflow_list;
1.9235 + do {
1.9236 + cur_overflow_list = observed_overflow_list;
1.9237 + if (cur_overflow_list != BUSY) {
1.9238 + p->set_mark(markOop(cur_overflow_list));
1.9239 + } else {
1.9240 + p->set_mark(NULL);
1.9241 + }
1.9242 + observed_overflow_list =
1.9243 + (oop) Atomic::cmpxchg_ptr(p, &_overflow_list, cur_overflow_list);
1.9244 + } while (cur_overflow_list != observed_overflow_list);
1.9245 +}
1.9246 +#undef BUSY
1.9247 +
1.9248 +// Single threaded
1.9249 +// General Note on GrowableArray: pushes may silently fail
1.9250 +// because we are (temporarily) out of C-heap for expanding
1.9251 +// the stack. The problem is quite ubiquitous and affects
1.9252 +// a lot of code in the JVM. The prudent thing for GrowableArray
1.9253 +// to do (for now) is to exit with an error. However, that may
1.9254 +// be too draconian in some cases because the caller may be
1.9255 +// able to recover without much harm. For such cases, we
1.9256 +// should probably introduce a "soft_push" method which returns
1.9257 +// an indication of success or failure with the assumption that
1.9258 +// the caller may be able to recover from a failure; code in
1.9259 +// the VM can then be changed, incrementally, to deal with such
1.9260 +// failures where possible, thus, incrementally hardening the VM
1.9261 +// in such low resource situations.
1.9262 +void CMSCollector::preserve_mark_work(oop p, markOop m) {
1.9263 + _preserved_oop_stack.push(p);
1.9264 + _preserved_mark_stack.push(m);
1.9265 + assert(m == p->mark(), "Mark word changed");
1.9266 + assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
1.9267 + "bijection");
1.9268 +}
1.9269 +
1.9270 +// Single threaded
1.9271 +void CMSCollector::preserve_mark_if_necessary(oop p) {
1.9272 + markOop m = p->mark();
1.9273 + if (m->must_be_preserved(p)) {
1.9274 + preserve_mark_work(p, m);
1.9275 + }
1.9276 +}
1.9277 +
1.9278 +void CMSCollector::par_preserve_mark_if_necessary(oop p) {
1.9279 + markOop m = p->mark();
1.9280 + if (m->must_be_preserved(p)) {
1.9281 + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1.9282 + // Even though we read the mark word without holding
1.9283 + // the lock, we are assured that it will not change
1.9284 + // because we "own" this oop, so no other thread can
1.9285 + // be trying to push it on the overflow list; see
1.9286 + // the assertion in preserve_mark_work() that checks
1.9287 + // that m == p->mark().
1.9288 + preserve_mark_work(p, m);
1.9289 + }
1.9290 +}
1.9291 +
1.9292 +// We should be able to do this multi-threaded,
1.9293 +// a chunk of stack being a task (this is
1.9294 +// correct because each oop only ever appears
1.9295 +// once in the overflow list. However, it's
1.9296 +// not very easy to completely overlap this with
1.9297 +// other operations, so will generally not be done
1.9298 +// until all work's been completed. Because we
1.9299 +// expect the preserved oop stack (set) to be small,
1.9300 +// it's probably fine to do this single-threaded.
1.9301 +// We can explore cleverer concurrent/overlapped/parallel
1.9302 +// processing of preserved marks if we feel the
1.9303 +// need for this in the future. Stack overflow should
1.9304 +// be so rare in practice and, when it happens, its
1.9305 +// effect on performance so great that this will
1.9306 +// likely just be in the noise anyway.
1.9307 +void CMSCollector::restore_preserved_marks_if_any() {
1.9308 + assert(SafepointSynchronize::is_at_safepoint(),
1.9309 + "world should be stopped");
1.9310 + assert(Thread::current()->is_ConcurrentGC_thread() ||
1.9311 + Thread::current()->is_VM_thread(),
1.9312 + "should be single-threaded");
1.9313 + assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
1.9314 + "bijection");
1.9315 +
1.9316 + while (!_preserved_oop_stack.is_empty()) {
1.9317 + oop p = _preserved_oop_stack.pop();
1.9318 + assert(p->is_oop(), "Should be an oop");
1.9319 + assert(_span.contains(p), "oop should be in _span");
1.9320 + assert(p->mark() == markOopDesc::prototype(),
1.9321 + "Set when taken from overflow list");
1.9322 + markOop m = _preserved_mark_stack.pop();
1.9323 + p->set_mark(m);
1.9324 + }
1.9325 + assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(),
1.9326 + "stacks were cleared above");
1.9327 +}
1.9328 +
1.9329 +#ifndef PRODUCT
1.9330 +bool CMSCollector::no_preserved_marks() const {
1.9331 + return _preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty();
1.9332 +}
1.9333 +#endif
1.9334 +
1.9335 +CMSAdaptiveSizePolicy* ASConcurrentMarkSweepGeneration::cms_size_policy() const
1.9336 +{
1.9337 + GenCollectedHeap* gch = (GenCollectedHeap*) GenCollectedHeap::heap();
1.9338 + CMSAdaptiveSizePolicy* size_policy =
1.9339 + (CMSAdaptiveSizePolicy*) gch->gen_policy()->size_policy();
1.9340 + assert(size_policy->is_gc_cms_adaptive_size_policy(),
1.9341 + "Wrong type for size policy");
1.9342 + return size_policy;
1.9343 +}
1.9344 +
1.9345 +void ASConcurrentMarkSweepGeneration::resize(size_t cur_promo_size,
1.9346 + size_t desired_promo_size) {
1.9347 + if (cur_promo_size < desired_promo_size) {
1.9348 + size_t expand_bytes = desired_promo_size - cur_promo_size;
1.9349 + if (PrintAdaptiveSizePolicy && Verbose) {
1.9350 + gclog_or_tty->print_cr(" ASConcurrentMarkSweepGeneration::resize "
1.9351 + "Expanding tenured generation by " SIZE_FORMAT " (bytes)",
1.9352 + expand_bytes);
1.9353 + }
1.9354 + expand(expand_bytes,
1.9355 + MinHeapDeltaBytes,
1.9356 + CMSExpansionCause::_adaptive_size_policy);
1.9357 + } else if (desired_promo_size < cur_promo_size) {
1.9358 + size_t shrink_bytes = cur_promo_size - desired_promo_size;
1.9359 + if (PrintAdaptiveSizePolicy && Verbose) {
1.9360 + gclog_or_tty->print_cr(" ASConcurrentMarkSweepGeneration::resize "
1.9361 + "Shrinking tenured generation by " SIZE_FORMAT " (bytes)",
1.9362 + shrink_bytes);
1.9363 + }
1.9364 + shrink(shrink_bytes);
1.9365 + }
1.9366 +}
1.9367 +
1.9368 +CMSGCAdaptivePolicyCounters* ASConcurrentMarkSweepGeneration::gc_adaptive_policy_counters() {
1.9369 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.9370 + CMSGCAdaptivePolicyCounters* counters =
1.9371 + (CMSGCAdaptivePolicyCounters*) gch->collector_policy()->counters();
1.9372 + assert(counters->kind() == GCPolicyCounters::CMSGCAdaptivePolicyCountersKind,
1.9373 + "Wrong kind of counters");
1.9374 + return counters;
1.9375 +}
1.9376 +
1.9377 +
1.9378 +void ASConcurrentMarkSweepGeneration::update_counters() {
1.9379 + if (UsePerfData) {
1.9380 + _space_counters->update_all();
1.9381 + _gen_counters->update_all();
1.9382 + CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
1.9383 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.9384 + CMSGCStats* gc_stats_l = (CMSGCStats*) gc_stats();
1.9385 + assert(gc_stats_l->kind() == GCStats::CMSGCStatsKind,
1.9386 + "Wrong gc statistics type");
1.9387 + counters->update_counters(gc_stats_l);
1.9388 + }
1.9389 +}
1.9390 +
1.9391 +void ASConcurrentMarkSweepGeneration::update_counters(size_t used) {
1.9392 + if (UsePerfData) {
1.9393 + _space_counters->update_used(used);
1.9394 + _space_counters->update_capacity();
1.9395 + _gen_counters->update_all();
1.9396 +
1.9397 + CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
1.9398 + GenCollectedHeap* gch = GenCollectedHeap::heap();
1.9399 + CMSGCStats* gc_stats_l = (CMSGCStats*) gc_stats();
1.9400 + assert(gc_stats_l->kind() == GCStats::CMSGCStatsKind,
1.9401 + "Wrong gc statistics type");
1.9402 + counters->update_counters(gc_stats_l);
1.9403 + }
1.9404 +}
1.9405 +
1.9406 +void ASConcurrentMarkSweepGeneration::shrink_by(size_t desired_bytes) {
1.9407 + assert_locked_or_safepoint(Heap_lock);
1.9408 + assert_lock_strong(freelistLock());
1.9409 + HeapWord* old_end = _cmsSpace->end();
1.9410 + HeapWord* unallocated_start = _cmsSpace->unallocated_block();
1.9411 + assert(old_end >= unallocated_start, "Miscalculation of unallocated_start");
1.9412 + FreeChunk* chunk_at_end = find_chunk_at_end();
1.9413 + if (chunk_at_end == NULL) {
1.9414 + // No room to shrink
1.9415 + if (PrintGCDetails && Verbose) {
1.9416 + gclog_or_tty->print_cr("No room to shrink: old_end "
1.9417 + PTR_FORMAT " unallocated_start " PTR_FORMAT
1.9418 + " chunk_at_end " PTR_FORMAT,
1.9419 + old_end, unallocated_start, chunk_at_end);
1.9420 + }
1.9421 + return;
1.9422 + } else {
1.9423 +
1.9424 + // Find the chunk at the end of the space and determine
1.9425 + // how much it can be shrunk.
1.9426 + size_t shrinkable_size_in_bytes = chunk_at_end->size();
1.9427 + size_t aligned_shrinkable_size_in_bytes =
1.9428 + align_size_down(shrinkable_size_in_bytes, os::vm_page_size());
1.9429 + assert(unallocated_start <= (HeapWord*) chunk_at_end->end(),
1.9430 + "Inconsistent chunk at end of space");
1.9431 + size_t bytes = MIN2(desired_bytes, aligned_shrinkable_size_in_bytes);
1.9432 + size_t word_size_before = heap_word_size(_virtual_space.committed_size());
1.9433 +
1.9434 + // Shrink the underlying space
1.9435 + _virtual_space.shrink_by(bytes);
1.9436 + if (PrintGCDetails && Verbose) {
1.9437 + gclog_or_tty->print_cr("ConcurrentMarkSweepGeneration::shrink_by:"
1.9438 + " desired_bytes " SIZE_FORMAT
1.9439 + " shrinkable_size_in_bytes " SIZE_FORMAT
1.9440 + " aligned_shrinkable_size_in_bytes " SIZE_FORMAT
1.9441 + " bytes " SIZE_FORMAT,
1.9442 + desired_bytes, shrinkable_size_in_bytes,
1.9443 + aligned_shrinkable_size_in_bytes, bytes);
1.9444 + gclog_or_tty->print_cr(" old_end " SIZE_FORMAT
1.9445 + " unallocated_start " SIZE_FORMAT,
1.9446 + old_end, unallocated_start);
1.9447 + }
1.9448 +
1.9449 + // If the space did shrink (shrinking is not guaranteed),
1.9450 + // shrink the chunk at the end by the appropriate amount.
1.9451 + if (((HeapWord*)_virtual_space.high()) < old_end) {
1.9452 + size_t new_word_size =
1.9453 + heap_word_size(_virtual_space.committed_size());
1.9454 +
1.9455 + // Have to remove the chunk from the dictionary because it is changing
1.9456 + // size and might be someplace elsewhere in the dictionary.
1.9457 +
1.9458 + // Get the chunk at end, shrink it, and put it
1.9459 + // back.
1.9460 + _cmsSpace->removeChunkFromDictionary(chunk_at_end);
1.9461 + size_t word_size_change = word_size_before - new_word_size;
1.9462 + size_t chunk_at_end_old_size = chunk_at_end->size();
1.9463 + assert(chunk_at_end_old_size >= word_size_change,
1.9464 + "Shrink is too large");
1.9465 + chunk_at_end->set_size(chunk_at_end_old_size -
1.9466 + word_size_change);
1.9467 + _cmsSpace->freed((HeapWord*) chunk_at_end->end(),
1.9468 + word_size_change);
1.9469 +
1.9470 + _cmsSpace->returnChunkToDictionary(chunk_at_end);
1.9471 +
1.9472 + MemRegion mr(_cmsSpace->bottom(), new_word_size);
1.9473 + _bts->resize(new_word_size); // resize the block offset shared array
1.9474 + Universe::heap()->barrier_set()->resize_covered_region(mr);
1.9475 + _cmsSpace->assert_locked();
1.9476 + _cmsSpace->set_end((HeapWord*)_virtual_space.high());
1.9477 +
1.9478 + NOT_PRODUCT(_cmsSpace->dictionary()->verify());
1.9479 +
1.9480 + // update the space and generation capacity counters
1.9481 + if (UsePerfData) {
1.9482 + _space_counters->update_capacity();
1.9483 + _gen_counters->update_all();
1.9484 + }
1.9485 +
1.9486 + if (Verbose && PrintGCDetails) {
1.9487 + size_t new_mem_size = _virtual_space.committed_size();
1.9488 + size_t old_mem_size = new_mem_size + bytes;
1.9489 + gclog_or_tty->print_cr("Shrinking %s from " SIZE_FORMAT "K by " SIZE_FORMAT "K to " SIZE_FORMAT "K",
1.9490 + name(), old_mem_size/K, bytes/K, new_mem_size/K);
1.9491 + }
1.9492 + }
1.9493 +
1.9494 + assert(_cmsSpace->unallocated_block() <= _cmsSpace->end(),
1.9495 + "Inconsistency at end of space");
1.9496 + assert(chunk_at_end->end() == (uintptr_t*) _cmsSpace->end(),
1.9497 + "Shrinking is inconsistent");
1.9498 + return;
1.9499 + }
1.9500 +}
1.9501 +// Transfer some number of overflown objects to usual marking
1.9502 +// stack. Return true if some objects were transferred.
1.9503 +bool MarkRefsIntoAndScanClosure::take_from_overflow_list() {
1.9504 + size_t num = MIN2((size_t)(_mark_stack->capacity() - _mark_stack->length())/4,
1.9505 + (size_t)ParGCDesiredObjsFromOverflowList);
1.9506 +
1.9507 + bool res = _collector->take_from_overflow_list(num, _mark_stack);
1.9508 + assert(_collector->overflow_list_is_empty() || res,
1.9509 + "If list is not empty, we should have taken something");
1.9510 + assert(!res || !_mark_stack->isEmpty(),
1.9511 + "If we took something, it should now be on our stack");
1.9512 + return res;
1.9513 +}
1.9514 +
1.9515 +size_t MarkDeadObjectsClosure::do_blk(HeapWord* addr) {
1.9516 + size_t res = _sp->block_size_no_stall(addr, _collector);
1.9517 + if (_sp->block_is_obj(addr)) {
1.9518 + if (_live_bit_map->isMarked(addr)) {
1.9519 + // It can't have been dead in a previous cycle
1.9520 + guarantee(!_dead_bit_map->isMarked(addr), "No resurrection!");
1.9521 + } else {
1.9522 + _dead_bit_map->mark(addr); // mark the dead object
1.9523 + }
1.9524 + }
1.9525 + // Could be 0, if the block size could not be computed without stalling.
1.9526 + return res;
1.9527 +}
1.9528 +
1.9529 +TraceCMSMemoryManagerStats::TraceCMSMemoryManagerStats(CMSCollector::CollectorState phase, GCCause::Cause cause): TraceMemoryManagerStats() {
1.9530 +
1.9531 + switch (phase) {
1.9532 + case CMSCollector::InitialMarking:
1.9533 + initialize(true /* fullGC */ ,
1.9534 + cause /* cause of the GC */,
1.9535 + true /* recordGCBeginTime */,
1.9536 + true /* recordPreGCUsage */,
1.9537 + false /* recordPeakUsage */,
1.9538 + false /* recordPostGCusage */,
1.9539 + true /* recordAccumulatedGCTime */,
1.9540 + false /* recordGCEndTime */,
1.9541 + false /* countCollection */ );
1.9542 + break;
1.9543 +
1.9544 + case CMSCollector::FinalMarking:
1.9545 + initialize(true /* fullGC */ ,
1.9546 + cause /* cause of the GC */,
1.9547 + false /* recordGCBeginTime */,
1.9548 + false /* recordPreGCUsage */,
1.9549 + false /* recordPeakUsage */,
1.9550 + false /* recordPostGCusage */,
1.9551 + true /* recordAccumulatedGCTime */,
1.9552 + false /* recordGCEndTime */,
1.9553 + false /* countCollection */ );
1.9554 + break;
1.9555 +
1.9556 + case CMSCollector::Sweeping:
1.9557 + initialize(true /* fullGC */ ,
1.9558 + cause /* cause of the GC */,
1.9559 + false /* recordGCBeginTime */,
1.9560 + false /* recordPreGCUsage */,
1.9561 + true /* recordPeakUsage */,
1.9562 + true /* recordPostGCusage */,
1.9563 + false /* recordAccumulatedGCTime */,
1.9564 + true /* recordGCEndTime */,
1.9565 + true /* countCollection */ );
1.9566 + break;
1.9567 +
1.9568 + default:
1.9569 + ShouldNotReachHere();
1.9570 + }
1.9571 +}
