Sun, 16 Mar 2008 21:57:25 -0700
6634032: CMS: Need CMSInitiatingPermOccupancyFraction for perm, divorcing from CMSInitiatingOccupancyFraction
Summary: The option CMSInitiatingPermOccupancyFraction now controls perm triggering threshold. Even though the actual value of the threshold has not yet been changed, so there is no change in policy, we now have the infrastructure in place for dynamically deciding when to collect the perm gen, an issue that will be addressed in the near future.
Reviewed-by: jmasa
1 /*
2 * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
20 * CA 95054 USA or visit www.sun.com if you need additional information or
21 * have any questions.
22 *
23 */
25 # include "incls/_precompiled.incl"
26 # include "incls/_concurrentMarkSweepGeneration.cpp.incl"
28 // statics
29 CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;
30 bool CMSCollector::_full_gc_requested = false;
32 //////////////////////////////////////////////////////////////////
33 // In support of CMS/VM thread synchronization
34 //////////////////////////////////////////////////////////////////
35 // We split use of the CGC_lock into 2 "levels".
36 // The low-level locking is of the usual CGC_lock monitor. We introduce
37 // a higher level "token" (hereafter "CMS token") built on top of the
38 // low level monitor (hereafter "CGC lock").
39 // The token-passing protocol gives priority to the VM thread. The
40 // CMS-lock doesn't provide any fairness guarantees, but clients
41 // should ensure that it is only held for very short, bounded
42 // durations.
43 //
44 // When either of the CMS thread or the VM thread is involved in
45 // collection operations during which it does not want the other
46 // thread to interfere, it obtains the CMS token.
47 //
48 // If either thread tries to get the token while the other has
49 // it, that thread waits. However, if the VM thread and CMS thread
50 // both want the token, then the VM thread gets priority while the
51 // CMS thread waits. This ensures, for instance, that the "concurrent"
52 // phases of the CMS thread's work do not block out the VM thread
53 // for long periods of time as the CMS thread continues to hog
54 // the token. (See bug 4616232).
55 //
56 // The baton-passing functions are, however, controlled by the
57 // flags _foregroundGCShouldWait and _foregroundGCIsActive,
58 // and here the low-level CMS lock, not the high level token,
59 // ensures mutual exclusion.
60 //
61 // Two important conditions that we have to satisfy:
62 // 1. if a thread does a low-level wait on the CMS lock, then it
63 // relinquishes the CMS token if it were holding that token
64 // when it acquired the low-level CMS lock.
65 // 2. any low-level notifications on the low-level lock
66 // should only be sent when a thread has relinquished the token.
67 //
68 // In the absence of either property, we'd have potential deadlock.
69 //
70 // We protect each of the CMS (concurrent and sequential) phases
71 // with the CMS _token_, not the CMS _lock_.
72 //
73 // The only code protected by CMS lock is the token acquisition code
74 // itself, see ConcurrentMarkSweepThread::[de]synchronize(), and the
75 // baton-passing code.
76 //
77 // Unfortunately, i couldn't come up with a good abstraction to factor and
78 // hide the naked CGC_lock manipulation in the baton-passing code
79 // further below. That's something we should try to do. Also, the proof
80 // of correctness of this 2-level locking scheme is far from obvious,
81 // and potentially quite slippery. We have an uneasy supsicion, for instance,
82 // that there may be a theoretical possibility of delay/starvation in the
83 // low-level lock/wait/notify scheme used for the baton-passing because of
84 // potential intereference with the priority scheme embodied in the
85 // CMS-token-passing protocol. See related comments at a CGC_lock->wait()
86 // invocation further below and marked with "XXX 20011219YSR".
87 // Indeed, as we note elsewhere, this may become yet more slippery
88 // in the presence of multiple CMS and/or multiple VM threads. XXX
90 class CMSTokenSync: public StackObj {
91 private:
92 bool _is_cms_thread;
93 public:
94 CMSTokenSync(bool is_cms_thread):
95 _is_cms_thread(is_cms_thread) {
96 assert(is_cms_thread == Thread::current()->is_ConcurrentGC_thread(),
97 "Incorrect argument to constructor");
98 ConcurrentMarkSweepThread::synchronize(_is_cms_thread);
99 }
101 ~CMSTokenSync() {
102 assert(_is_cms_thread ?
103 ConcurrentMarkSweepThread::cms_thread_has_cms_token() :
104 ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
105 "Incorrect state");
106 ConcurrentMarkSweepThread::desynchronize(_is_cms_thread);
107 }
108 };
110 // Convenience class that does a CMSTokenSync, and then acquires
111 // upto three locks.
112 class CMSTokenSyncWithLocks: public CMSTokenSync {
113 private:
114 // Note: locks are acquired in textual declaration order
115 // and released in the opposite order
116 MutexLockerEx _locker1, _locker2, _locker3;
117 public:
118 CMSTokenSyncWithLocks(bool is_cms_thread, Mutex* mutex1,
119 Mutex* mutex2 = NULL, Mutex* mutex3 = NULL):
120 CMSTokenSync(is_cms_thread),
121 _locker1(mutex1, Mutex::_no_safepoint_check_flag),
122 _locker2(mutex2, Mutex::_no_safepoint_check_flag),
123 _locker3(mutex3, Mutex::_no_safepoint_check_flag)
124 { }
125 };
128 // Wrapper class to temporarily disable icms during a foreground cms collection.
129 class ICMSDisabler: public StackObj {
130 public:
131 // The ctor disables icms and wakes up the thread so it notices the change;
132 // the dtor re-enables icms. Note that the CMSCollector methods will check
133 // CMSIncrementalMode.
134 ICMSDisabler() { CMSCollector::disable_icms(); CMSCollector::start_icms(); }
135 ~ICMSDisabler() { CMSCollector::enable_icms(); }
136 };
138 //////////////////////////////////////////////////////////////////
139 // Concurrent Mark-Sweep Generation /////////////////////////////
140 //////////////////////////////////////////////////////////////////
142 NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;)
144 // This struct contains per-thread things necessary to support parallel
145 // young-gen collection.
146 class CMSParGCThreadState: public CHeapObj {
147 public:
148 CFLS_LAB lab;
149 PromotionInfo promo;
151 // Constructor.
152 CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) {
153 promo.setSpace(cfls);
154 }
155 };
157 ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration(
158 ReservedSpace rs, size_t initial_byte_size, int level,
159 CardTableRS* ct, bool use_adaptive_freelists,
160 FreeBlockDictionary::DictionaryChoice dictionaryChoice) :
161 CardGeneration(rs, initial_byte_size, level, ct),
162 _dilatation_factor(((double)MinChunkSize)/((double)(oopDesc::header_size()))),
163 _debug_collection_type(Concurrent_collection_type)
164 {
165 HeapWord* bottom = (HeapWord*) _virtual_space.low();
166 HeapWord* end = (HeapWord*) _virtual_space.high();
168 _direct_allocated_words = 0;
169 NOT_PRODUCT(
170 _numObjectsPromoted = 0;
171 _numWordsPromoted = 0;
172 _numObjectsAllocated = 0;
173 _numWordsAllocated = 0;
174 )
176 _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end),
177 use_adaptive_freelists,
178 dictionaryChoice);
179 NOT_PRODUCT(debug_cms_space = _cmsSpace;)
180 if (_cmsSpace == NULL) {
181 vm_exit_during_initialization(
182 "CompactibleFreeListSpace allocation failure");
183 }
184 _cmsSpace->_gen = this;
186 _gc_stats = new CMSGCStats();
188 // Verify the assumption that FreeChunk::_prev and OopDesc::_klass
189 // offsets match. The ability to tell free chunks from objects
190 // depends on this property.
191 debug_only(
192 FreeChunk* junk = NULL;
193 assert(junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
194 "Offset of FreeChunk::_prev within FreeChunk must match"
195 " that of OopDesc::_klass within OopDesc");
196 )
197 if (ParallelGCThreads > 0) {
198 typedef CMSParGCThreadState* CMSParGCThreadStatePtr;
199 _par_gc_thread_states =
200 NEW_C_HEAP_ARRAY(CMSParGCThreadStatePtr, ParallelGCThreads);
201 if (_par_gc_thread_states == NULL) {
202 vm_exit_during_initialization("Could not allocate par gc structs");
203 }
204 for (uint i = 0; i < ParallelGCThreads; i++) {
205 _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace());
206 if (_par_gc_thread_states[i] == NULL) {
207 vm_exit_during_initialization("Could not allocate par gc structs");
208 }
209 }
210 } else {
211 _par_gc_thread_states = NULL;
212 }
213 _incremental_collection_failed = false;
214 // The "dilatation_factor" is the expansion that can occur on
215 // account of the fact that the minimum object size in the CMS
216 // generation may be larger than that in, say, a contiguous young
217 // generation.
218 // Ideally, in the calculation below, we'd compute the dilatation
219 // factor as: MinChunkSize/(promoting_gen's min object size)
220 // Since we do not have such a general query interface for the
221 // promoting generation, we'll instead just use the mimimum
222 // object size (which today is a header's worth of space);
223 // note that all arithmetic is in units of HeapWords.
224 assert(MinChunkSize >= oopDesc::header_size(), "just checking");
225 assert(_dilatation_factor >= 1.0, "from previous assert");
226 }
229 // The field "_initiating_occupancy" represents the occupancy percentage
230 // at which we trigger a new collection cycle. Unless explicitly specified
231 // via CMSInitiating[Perm]OccupancyFraction (argument "io" below), it
232 // is calculated by:
233 //
234 // Let "f" be MinHeapFreeRatio in
235 //
236 // _intiating_occupancy = 100-f +
237 // f * (CMSTrigger[Perm]Ratio/100)
238 // where CMSTrigger[Perm]Ratio is the argument "tr" below.
239 //
240 // That is, if we assume the heap is at its desired maximum occupancy at the
241 // end of a collection, we let CMSTrigger[Perm]Ratio of the (purported) free
242 // space be allocated before initiating a new collection cycle.
243 //
244 void ConcurrentMarkSweepGeneration::init_initiating_occupancy(intx io, intx tr) {
245 assert(io <= 100 && tr >= 0 && tr <= 100, "Check the arguments");
246 if (io >= 0) {
247 _initiating_occupancy = (double)io / 100.0;
248 } else {
249 _initiating_occupancy = ((100 - MinHeapFreeRatio) +
250 (double)(tr * MinHeapFreeRatio) / 100.0)
251 / 100.0;
252 }
253 }
256 void ConcurrentMarkSweepGeneration::ref_processor_init() {
257 assert(collector() != NULL, "no collector");
258 collector()->ref_processor_init();
259 }
261 void CMSCollector::ref_processor_init() {
262 if (_ref_processor == NULL) {
263 // Allocate and initialize a reference processor
264 _ref_processor = ReferenceProcessor::create_ref_processor(
265 _span, // span
266 _cmsGen->refs_discovery_is_atomic(), // atomic_discovery
267 _cmsGen->refs_discovery_is_mt(), // mt_discovery
268 &_is_alive_closure,
269 ParallelGCThreads,
270 ParallelRefProcEnabled);
271 // Initialize the _ref_processor field of CMSGen
272 _cmsGen->set_ref_processor(_ref_processor);
274 // Allocate a dummy ref processor for perm gen.
275 ReferenceProcessor* rp2 = new ReferenceProcessor();
276 if (rp2 == NULL) {
277 vm_exit_during_initialization("Could not allocate ReferenceProcessor object");
278 }
279 _permGen->set_ref_processor(rp2);
280 }
281 }
283 CMSAdaptiveSizePolicy* CMSCollector::size_policy() {
284 GenCollectedHeap* gch = GenCollectedHeap::heap();
285 assert(gch->kind() == CollectedHeap::GenCollectedHeap,
286 "Wrong type of heap");
287 CMSAdaptiveSizePolicy* sp = (CMSAdaptiveSizePolicy*)
288 gch->gen_policy()->size_policy();
289 assert(sp->is_gc_cms_adaptive_size_policy(),
290 "Wrong type of size policy");
291 return sp;
292 }
294 CMSGCAdaptivePolicyCounters* CMSCollector::gc_adaptive_policy_counters() {
295 CMSGCAdaptivePolicyCounters* results =
296 (CMSGCAdaptivePolicyCounters*) collector_policy()->counters();
297 assert(
298 results->kind() == GCPolicyCounters::CMSGCAdaptivePolicyCountersKind,
299 "Wrong gc policy counter kind");
300 return results;
301 }
304 void ConcurrentMarkSweepGeneration::initialize_performance_counters() {
306 const char* gen_name = "old";
308 // Generation Counters - generation 1, 1 subspace
309 _gen_counters = new GenerationCounters(gen_name, 1, 1, &_virtual_space);
311 _space_counters = new GSpaceCounters(gen_name, 0,
312 _virtual_space.reserved_size(),
313 this, _gen_counters);
314 }
316 CMSStats::CMSStats(ConcurrentMarkSweepGeneration* cms_gen, unsigned int alpha):
317 _cms_gen(cms_gen)
318 {
319 assert(alpha <= 100, "bad value");
320 _saved_alpha = alpha;
322 // Initialize the alphas to the bootstrap value of 100.
323 _gc0_alpha = _cms_alpha = 100;
325 _cms_begin_time.update();
326 _cms_end_time.update();
328 _gc0_duration = 0.0;
329 _gc0_period = 0.0;
330 _gc0_promoted = 0;
332 _cms_duration = 0.0;
333 _cms_period = 0.0;
334 _cms_allocated = 0;
336 _cms_used_at_gc0_begin = 0;
337 _cms_used_at_gc0_end = 0;
338 _allow_duty_cycle_reduction = false;
339 _valid_bits = 0;
340 _icms_duty_cycle = CMSIncrementalDutyCycle;
341 }
343 // If promotion failure handling is on use
344 // the padded average size of the promotion for each
345 // young generation collection.
346 double CMSStats::time_until_cms_gen_full() const {
347 size_t cms_free = _cms_gen->cmsSpace()->free();
348 GenCollectedHeap* gch = GenCollectedHeap::heap();
349 size_t expected_promotion = gch->get_gen(0)->capacity();
350 if (HandlePromotionFailure) {
351 expected_promotion = MIN2(
352 (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average(),
353 expected_promotion);
354 }
355 if (cms_free > expected_promotion) {
356 // Start a cms collection if there isn't enough space to promote
357 // for the next minor collection. Use the padded average as
358 // a safety factor.
359 cms_free -= expected_promotion;
361 // Adjust by the safety factor.
362 double cms_free_dbl = (double)cms_free;
363 cms_free_dbl = cms_free_dbl * (100.0 - CMSIncrementalSafetyFactor) / 100.0;
365 if (PrintGCDetails && Verbose) {
366 gclog_or_tty->print_cr("CMSStats::time_until_cms_gen_full: cms_free "
367 SIZE_FORMAT " expected_promotion " SIZE_FORMAT,
368 cms_free, expected_promotion);
369 gclog_or_tty->print_cr(" cms_free_dbl %f cms_consumption_rate %f",
370 cms_free_dbl, cms_consumption_rate() + 1.0);
371 }
372 // Add 1 in case the consumption rate goes to zero.
373 return cms_free_dbl / (cms_consumption_rate() + 1.0);
374 }
375 return 0.0;
376 }
378 // Compare the duration of the cms collection to the
379 // time remaining before the cms generation is empty.
380 // Note that the time from the start of the cms collection
381 // to the start of the cms sweep (less than the total
382 // duration of the cms collection) can be used. This
383 // has been tried and some applications experienced
384 // promotion failures early in execution. This was
385 // possibly because the averages were not accurate
386 // enough at the beginning.
387 double CMSStats::time_until_cms_start() const {
388 // We add "gc0_period" to the "work" calculation
389 // below because this query is done (mostly) at the
390 // end of a scavenge, so we need to conservatively
391 // account for that much possible delay
392 // in the query so as to avoid concurrent mode failures
393 // due to starting the collection just a wee bit too
394 // late.
395 double work = cms_duration() + gc0_period();
396 double deadline = time_until_cms_gen_full();
397 if (work > deadline) {
398 if (Verbose && PrintGCDetails) {
399 gclog_or_tty->print(
400 " CMSCollector: collect because of anticipated promotion "
401 "before full %3.7f + %3.7f > %3.7f ", cms_duration(),
402 gc0_period(), time_until_cms_gen_full());
403 }
404 return 0.0;
405 }
406 return work - deadline;
407 }
409 // Return a duty cycle based on old_duty_cycle and new_duty_cycle, limiting the
410 // amount of change to prevent wild oscillation.
411 unsigned int CMSStats::icms_damped_duty_cycle(unsigned int old_duty_cycle,
412 unsigned int new_duty_cycle) {
413 assert(old_duty_cycle <= 100, "bad input value");
414 assert(new_duty_cycle <= 100, "bad input value");
416 // Note: use subtraction with caution since it may underflow (values are
417 // unsigned). Addition is safe since we're in the range 0-100.
418 unsigned int damped_duty_cycle = new_duty_cycle;
419 if (new_duty_cycle < old_duty_cycle) {
420 const unsigned int largest_delta = MAX2(old_duty_cycle / 4, 5U);
421 if (new_duty_cycle + largest_delta < old_duty_cycle) {
422 damped_duty_cycle = old_duty_cycle - largest_delta;
423 }
424 } else if (new_duty_cycle > old_duty_cycle) {
425 const unsigned int largest_delta = MAX2(old_duty_cycle / 4, 15U);
426 if (new_duty_cycle > old_duty_cycle + largest_delta) {
427 damped_duty_cycle = MIN2(old_duty_cycle + largest_delta, 100U);
428 }
429 }
430 assert(damped_duty_cycle <= 100, "invalid duty cycle computed");
432 if (CMSTraceIncrementalPacing) {
433 gclog_or_tty->print(" [icms_damped_duty_cycle(%d,%d) = %d] ",
434 old_duty_cycle, new_duty_cycle, damped_duty_cycle);
435 }
436 return damped_duty_cycle;
437 }
439 unsigned int CMSStats::icms_update_duty_cycle_impl() {
440 assert(CMSIncrementalPacing && valid(),
441 "should be handled in icms_update_duty_cycle()");
443 double cms_time_so_far = cms_timer().seconds();
444 double scaled_duration = cms_duration_per_mb() * _cms_used_at_gc0_end / M;
445 double scaled_duration_remaining = fabsd(scaled_duration - cms_time_so_far);
447 // Avoid division by 0.
448 double time_until_full = MAX2(time_until_cms_gen_full(), 0.01);
449 double duty_cycle_dbl = 100.0 * scaled_duration_remaining / time_until_full;
451 unsigned int new_duty_cycle = MIN2((unsigned int)duty_cycle_dbl, 100U);
452 if (new_duty_cycle > _icms_duty_cycle) {
453 // Avoid very small duty cycles (1 or 2); 0 is allowed.
454 if (new_duty_cycle > 2) {
455 _icms_duty_cycle = icms_damped_duty_cycle(_icms_duty_cycle,
456 new_duty_cycle);
457 }
458 } else if (_allow_duty_cycle_reduction) {
459 // The duty cycle is reduced only once per cms cycle (see record_cms_end()).
460 new_duty_cycle = icms_damped_duty_cycle(_icms_duty_cycle, new_duty_cycle);
461 // Respect the minimum duty cycle.
462 unsigned int min_duty_cycle = (unsigned int)CMSIncrementalDutyCycleMin;
463 _icms_duty_cycle = MAX2(new_duty_cycle, min_duty_cycle);
464 }
466 if (PrintGCDetails || CMSTraceIncrementalPacing) {
467 gclog_or_tty->print(" icms_dc=%d ", _icms_duty_cycle);
468 }
470 _allow_duty_cycle_reduction = false;
471 return _icms_duty_cycle;
472 }
474 #ifndef PRODUCT
475 void CMSStats::print_on(outputStream *st) const {
476 st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);
477 st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,
478 gc0_duration(), gc0_period(), gc0_promoted());
479 st->print(",cms_dur=%g,cms_dur_per_mb=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,
480 cms_duration(), cms_duration_per_mb(),
481 cms_period(), cms_allocated());
482 st->print(",cms_since_beg=%g,cms_since_end=%g",
483 cms_time_since_begin(), cms_time_since_end());
484 st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,
485 _cms_used_at_gc0_begin, _cms_used_at_gc0_end);
486 if (CMSIncrementalMode) {
487 st->print(",dc=%d", icms_duty_cycle());
488 }
490 if (valid()) {
491 st->print(",promo_rate=%g,cms_alloc_rate=%g",
492 promotion_rate(), cms_allocation_rate());
493 st->print(",cms_consumption_rate=%g,time_until_full=%g",
494 cms_consumption_rate(), time_until_cms_gen_full());
495 }
496 st->print(" ");
497 }
498 #endif // #ifndef PRODUCT
500 CMSCollector::CollectorState CMSCollector::_collectorState =
501 CMSCollector::Idling;
502 bool CMSCollector::_foregroundGCIsActive = false;
503 bool CMSCollector::_foregroundGCShouldWait = false;
505 CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
506 ConcurrentMarkSweepGeneration* permGen,
507 CardTableRS* ct,
508 ConcurrentMarkSweepPolicy* cp):
509 _cmsGen(cmsGen),
510 _permGen(permGen),
511 _ct(ct),
512 _ref_processor(NULL), // will be set later
513 _conc_workers(NULL), // may be set later
514 _abort_preclean(false),
515 _start_sampling(false),
516 _between_prologue_and_epilogue(false),
517 _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
518 _perm_gen_verify_bit_map(0, -1 /* no mutex */, "No_lock"),
519 _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize),
520 -1 /* lock-free */, "No_lock" /* dummy */),
521 _modUnionClosure(&_modUnionTable),
522 _modUnionClosurePar(&_modUnionTable),
523 _is_alive_closure(&_markBitMap),
524 _restart_addr(NULL),
525 _overflow_list(NULL),
526 _preserved_oop_stack(NULL),
527 _preserved_mark_stack(NULL),
528 _stats(cmsGen),
529 _eden_chunk_array(NULL), // may be set in ctor body
530 _eden_chunk_capacity(0), // -- ditto --
531 _eden_chunk_index(0), // -- ditto --
532 _survivor_plab_array(NULL), // -- ditto --
533 _survivor_chunk_array(NULL), // -- ditto --
534 _survivor_chunk_capacity(0), // -- ditto --
535 _survivor_chunk_index(0), // -- ditto --
536 _ser_pmc_preclean_ovflw(0),
537 _ser_pmc_remark_ovflw(0),
538 _par_pmc_remark_ovflw(0),
539 _ser_kac_ovflw(0),
540 _par_kac_ovflw(0),
541 #ifndef PRODUCT
542 _num_par_pushes(0),
543 #endif
544 _collection_count_start(0),
545 _verifying(false),
546 _icms_start_limit(NULL),
547 _icms_stop_limit(NULL),
548 _verification_mark_bm(0, Mutex::leaf + 1, "CMS_verification_mark_bm_lock"),
549 _completed_initialization(false),
550 _collector_policy(cp),
551 _should_unload_classes(false),
552 _concurrent_cycles_since_last_unload(0),
553 _sweep_estimate(CMS_SweepWeight, CMS_SweepPadding)
554 {
555 if (ExplicitGCInvokesConcurrentAndUnloadsClasses) {
556 ExplicitGCInvokesConcurrent = true;
557 }
558 // Now expand the span and allocate the collection support structures
559 // (MUT, marking bit map etc.) to cover both generations subject to
560 // collection.
562 // First check that _permGen is adjacent to _cmsGen and above it.
563 assert( _cmsGen->reserved().word_size() > 0
564 && _permGen->reserved().word_size() > 0,
565 "generations should not be of zero size");
566 assert(_cmsGen->reserved().intersection(_permGen->reserved()).is_empty(),
567 "_cmsGen and _permGen should not overlap");
568 assert(_cmsGen->reserved().end() == _permGen->reserved().start(),
569 "_cmsGen->end() different from _permGen->start()");
571 // For use by dirty card to oop closures.
572 _cmsGen->cmsSpace()->set_collector(this);
573 _permGen->cmsSpace()->set_collector(this);
575 // Adjust my span to cover old (cms) gen and perm gen
576 _span = _cmsGen->reserved()._union(_permGen->reserved());
577 // Initialize the span of is_alive_closure
578 _is_alive_closure.set_span(_span);
580 // Allocate MUT and marking bit map
581 {
582 MutexLockerEx x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag);
583 if (!_markBitMap.allocate(_span)) {
584 warning("Failed to allocate CMS Bit Map");
585 return;
586 }
587 assert(_markBitMap.covers(_span), "_markBitMap inconsistency?");
588 }
589 {
590 _modUnionTable.allocate(_span);
591 assert(_modUnionTable.covers(_span), "_modUnionTable inconsistency?");
592 }
594 if (!_markStack.allocate(CMSMarkStackSize)) {
595 warning("Failed to allocate CMS Marking Stack");
596 return;
597 }
598 if (!_revisitStack.allocate(CMSRevisitStackSize)) {
599 warning("Failed to allocate CMS Revisit Stack");
600 return;
601 }
603 // Support for multi-threaded concurrent phases
604 if (ParallelGCThreads > 0 && CMSConcurrentMTEnabled) {
605 if (FLAG_IS_DEFAULT(ParallelCMSThreads)) {
606 // just for now
607 FLAG_SET_DEFAULT(ParallelCMSThreads, (ParallelGCThreads + 3)/4);
608 }
609 if (ParallelCMSThreads > 1) {
610 _conc_workers = new YieldingFlexibleWorkGang("Parallel CMS Threads",
611 ParallelCMSThreads, true);
612 if (_conc_workers == NULL) {
613 warning("GC/CMS: _conc_workers allocation failure: "
614 "forcing -CMSConcurrentMTEnabled");
615 CMSConcurrentMTEnabled = false;
616 }
617 } else {
618 CMSConcurrentMTEnabled = false;
619 }
620 }
621 if (!CMSConcurrentMTEnabled) {
622 ParallelCMSThreads = 0;
623 } else {
624 // Turn off CMSCleanOnEnter optimization temporarily for
625 // the MT case where it's not fixed yet; see 6178663.
626 CMSCleanOnEnter = false;
627 }
628 assert((_conc_workers != NULL) == (ParallelCMSThreads > 1),
629 "Inconsistency");
631 // Parallel task queues; these are shared for the
632 // concurrent and stop-world phases of CMS, but
633 // are not shared with parallel scavenge (ParNew).
634 {
635 uint i;
636 uint num_queues = (uint) MAX2(ParallelGCThreads, ParallelCMSThreads);
638 if ((CMSParallelRemarkEnabled || CMSConcurrentMTEnabled
639 || ParallelRefProcEnabled)
640 && num_queues > 0) {
641 _task_queues = new OopTaskQueueSet(num_queues);
642 if (_task_queues == NULL) {
643 warning("task_queues allocation failure.");
644 return;
645 }
646 _hash_seed = NEW_C_HEAP_ARRAY(int, num_queues);
647 if (_hash_seed == NULL) {
648 warning("_hash_seed array allocation failure");
649 return;
650 }
652 // XXX use a global constant instead of 64!
653 typedef struct OopTaskQueuePadded {
654 OopTaskQueue work_queue;
655 char pad[64 - sizeof(OopTaskQueue)]; // prevent false sharing
656 } OopTaskQueuePadded;
658 for (i = 0; i < num_queues; i++) {
659 OopTaskQueuePadded *q_padded = new OopTaskQueuePadded();
660 if (q_padded == NULL) {
661 warning("work_queue allocation failure.");
662 return;
663 }
664 _task_queues->register_queue(i, &q_padded->work_queue);
665 }
666 for (i = 0; i < num_queues; i++) {
667 _task_queues->queue(i)->initialize();
668 _hash_seed[i] = 17; // copied from ParNew
669 }
670 }
671 }
673 _cmsGen ->init_initiating_occupancy(CMSInitiatingOccupancyFraction, CMSTriggerRatio);
674 _permGen->init_initiating_occupancy(CMSInitiatingPermOccupancyFraction, CMSTriggerPermRatio);
676 // Clip CMSBootstrapOccupancy between 0 and 100.
677 _bootstrap_occupancy = ((double)MIN2((uintx)100, MAX2((uintx)0, CMSBootstrapOccupancy)))
678 /(double)100;
680 _full_gcs_since_conc_gc = 0;
682 // Now tell CMS generations the identity of their collector
683 ConcurrentMarkSweepGeneration::set_collector(this);
685 // Create & start a CMS thread for this CMS collector
686 _cmsThread = ConcurrentMarkSweepThread::start(this);
687 assert(cmsThread() != NULL, "CMS Thread should have been created");
688 assert(cmsThread()->collector() == this,
689 "CMS Thread should refer to this gen");
690 assert(CGC_lock != NULL, "Where's the CGC_lock?");
692 // Support for parallelizing young gen rescan
693 GenCollectedHeap* gch = GenCollectedHeap::heap();
694 _young_gen = gch->prev_gen(_cmsGen);
695 if (gch->supports_inline_contig_alloc()) {
696 _top_addr = gch->top_addr();
697 _end_addr = gch->end_addr();
698 assert(_young_gen != NULL, "no _young_gen");
699 _eden_chunk_index = 0;
700 _eden_chunk_capacity = (_young_gen->max_capacity()+CMSSamplingGrain)/CMSSamplingGrain;
701 _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity);
702 if (_eden_chunk_array == NULL) {
703 _eden_chunk_capacity = 0;
704 warning("GC/CMS: _eden_chunk_array allocation failure");
705 }
706 }
707 assert(_eden_chunk_array != NULL || _eden_chunk_capacity == 0, "Error");
709 // Support for parallelizing survivor space rescan
710 if (CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) {
711 size_t max_plab_samples = MaxNewSize/((SurvivorRatio+2)*MinTLABSize);
712 _survivor_plab_array = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads);
713 _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, 2*max_plab_samples);
714 _cursor = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads);
715 if (_survivor_plab_array == NULL || _survivor_chunk_array == NULL
716 || _cursor == NULL) {
717 warning("Failed to allocate survivor plab/chunk array");
718 if (_survivor_plab_array != NULL) {
719 FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array);
720 _survivor_plab_array = NULL;
721 }
722 if (_survivor_chunk_array != NULL) {
723 FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array);
724 _survivor_chunk_array = NULL;
725 }
726 if (_cursor != NULL) {
727 FREE_C_HEAP_ARRAY(size_t, _cursor);
728 _cursor = NULL;
729 }
730 } else {
731 _survivor_chunk_capacity = 2*max_plab_samples;
732 for (uint i = 0; i < ParallelGCThreads; i++) {
733 HeapWord** vec = NEW_C_HEAP_ARRAY(HeapWord*, max_plab_samples);
734 if (vec == NULL) {
735 warning("Failed to allocate survivor plab array");
736 for (int j = i; j > 0; j--) {
737 FREE_C_HEAP_ARRAY(HeapWord*, _survivor_plab_array[j-1].array());
738 }
739 FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array);
740 FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array);
741 _survivor_plab_array = NULL;
742 _survivor_chunk_array = NULL;
743 _survivor_chunk_capacity = 0;
744 break;
745 } else {
746 ChunkArray* cur =
747 ::new (&_survivor_plab_array[i]) ChunkArray(vec,
748 max_plab_samples);
749 assert(cur->end() == 0, "Should be 0");
750 assert(cur->array() == vec, "Should be vec");
751 assert(cur->capacity() == max_plab_samples, "Error");
752 }
753 }
754 }
755 }
756 assert( ( _survivor_plab_array != NULL
757 && _survivor_chunk_array != NULL)
758 || ( _survivor_chunk_capacity == 0
759 && _survivor_chunk_index == 0),
760 "Error");
762 // Choose what strong roots should be scanned depending on verification options
763 // and perm gen collection mode.
764 if (!CMSClassUnloadingEnabled) {
765 // If class unloading is disabled we want to include all classes into the root set.
766 add_root_scanning_option(SharedHeap::SO_AllClasses);
767 } else {
768 add_root_scanning_option(SharedHeap::SO_SystemClasses);
769 }
771 NOT_PRODUCT(_overflow_counter = CMSMarkStackOverflowInterval;)
772 _gc_counters = new CollectorCounters("CMS", 1);
773 _completed_initialization = true;
774 _sweep_timer.start(); // start of time
775 }
777 const char* ConcurrentMarkSweepGeneration::name() const {
778 return "concurrent mark-sweep generation";
779 }
780 void ConcurrentMarkSweepGeneration::update_counters() {
781 if (UsePerfData) {
782 _space_counters->update_all();
783 _gen_counters->update_all();
784 }
785 }
787 // this is an optimized version of update_counters(). it takes the
788 // used value as a parameter rather than computing it.
789 //
790 void ConcurrentMarkSweepGeneration::update_counters(size_t used) {
791 if (UsePerfData) {
792 _space_counters->update_used(used);
793 _space_counters->update_capacity();
794 _gen_counters->update_all();
795 }
796 }
798 void ConcurrentMarkSweepGeneration::print() const {
799 Generation::print();
800 cmsSpace()->print();
801 }
803 #ifndef PRODUCT
804 void ConcurrentMarkSweepGeneration::print_statistics() {
805 cmsSpace()->printFLCensus(0);
806 }
807 #endif
809 void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) {
810 GenCollectedHeap* gch = GenCollectedHeap::heap();
811 if (PrintGCDetails) {
812 if (Verbose) {
813 gclog_or_tty->print(" [%d %s-%s: "SIZE_FORMAT"("SIZE_FORMAT")]",
814 level(), short_name(), s, used(), capacity());
815 } else {
816 gclog_or_tty->print(" [%d %s-%s: "SIZE_FORMAT"K("SIZE_FORMAT"K)]",
817 level(), short_name(), s, used() / K, capacity() / K);
818 }
819 }
820 if (Verbose) {
821 gclog_or_tty->print(" "SIZE_FORMAT"("SIZE_FORMAT")",
822 gch->used(), gch->capacity());
823 } else {
824 gclog_or_tty->print(" "SIZE_FORMAT"K("SIZE_FORMAT"K)",
825 gch->used() / K, gch->capacity() / K);
826 }
827 }
829 size_t
830 ConcurrentMarkSweepGeneration::contiguous_available() const {
831 // dld proposes an improvement in precision here. If the committed
832 // part of the space ends in a free block we should add that to
833 // uncommitted size in the calculation below. Will make this
834 // change later, staying with the approximation below for the
835 // time being. -- ysr.
836 return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
837 }
839 size_t
840 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
841 return _cmsSpace->max_alloc_in_words() * HeapWordSize;
842 }
844 size_t ConcurrentMarkSweepGeneration::max_available() const {
845 return free() + _virtual_space.uncommitted_size();
846 }
848 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(
849 size_t max_promotion_in_bytes,
850 bool younger_handles_promotion_failure) const {
852 // This is the most conservative test. Full promotion is
853 // guaranteed if this is used. The multiplicative factor is to
854 // account for the worst case "dilatation".
855 double adjusted_max_promo_bytes = _dilatation_factor * max_promotion_in_bytes;
856 if (adjusted_max_promo_bytes > (double)max_uintx) { // larger than size_t
857 adjusted_max_promo_bytes = (double)max_uintx;
858 }
859 bool result = (max_contiguous_available() >= (size_t)adjusted_max_promo_bytes);
861 if (younger_handles_promotion_failure && !result) {
862 // Full promotion is not guaranteed because fragmentation
863 // of the cms generation can prevent the full promotion.
864 result = (max_available() >= (size_t)adjusted_max_promo_bytes);
866 if (!result) {
867 // With promotion failure handling the test for the ability
868 // to support the promotion does not have to be guaranteed.
869 // Use an average of the amount promoted.
870 result = max_available() >= (size_t)
871 gc_stats()->avg_promoted()->padded_average();
872 if (PrintGC && Verbose && result) {
873 gclog_or_tty->print_cr(
874 "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
875 " max_available: " SIZE_FORMAT
876 " avg_promoted: " SIZE_FORMAT,
877 max_available(), (size_t)
878 gc_stats()->avg_promoted()->padded_average());
879 }
880 } else {
881 if (PrintGC && Verbose) {
882 gclog_or_tty->print_cr(
883 "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
884 " max_available: " SIZE_FORMAT
885 " adj_max_promo_bytes: " SIZE_FORMAT,
886 max_available(), (size_t)adjusted_max_promo_bytes);
887 }
888 }
889 } else {
890 if (PrintGC && Verbose) {
891 gclog_or_tty->print_cr(
892 "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
893 " contiguous_available: " SIZE_FORMAT
894 " adj_max_promo_bytes: " SIZE_FORMAT,
895 max_contiguous_available(), (size_t)adjusted_max_promo_bytes);
896 }
897 }
898 return result;
899 }
901 CompactibleSpace*
902 ConcurrentMarkSweepGeneration::first_compaction_space() const {
903 return _cmsSpace;
904 }
906 void ConcurrentMarkSweepGeneration::reset_after_compaction() {
907 // Clear the promotion information. These pointers can be adjusted
908 // along with all the other pointers into the heap but
909 // compaction is expected to be a rare event with
910 // a heap using cms so don't do it without seeing the need.
911 if (ParallelGCThreads > 0) {
912 for (uint i = 0; i < ParallelGCThreads; i++) {
913 _par_gc_thread_states[i]->promo.reset();
914 }
915 }
916 }
918 void ConcurrentMarkSweepGeneration::space_iterate(SpaceClosure* blk, bool usedOnly) {
919 blk->do_space(_cmsSpace);
920 }
922 void ConcurrentMarkSweepGeneration::compute_new_size() {
923 assert_locked_or_safepoint(Heap_lock);
925 // If incremental collection failed, we just want to expand
926 // to the limit.
927 if (incremental_collection_failed()) {
928 clear_incremental_collection_failed();
929 grow_to_reserved();
930 return;
931 }
933 size_t expand_bytes = 0;
934 double free_percentage = ((double) free()) / capacity();
935 double desired_free_percentage = (double) MinHeapFreeRatio / 100;
936 double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
938 // compute expansion delta needed for reaching desired free percentage
939 if (free_percentage < desired_free_percentage) {
940 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
941 assert(desired_capacity >= capacity(), "invalid expansion size");
942 expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
943 }
944 if (expand_bytes > 0) {
945 if (PrintGCDetails && Verbose) {
946 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
947 gclog_or_tty->print_cr("\nFrom compute_new_size: ");
948 gclog_or_tty->print_cr(" Free fraction %f", free_percentage);
949 gclog_or_tty->print_cr(" Desired free fraction %f",
950 desired_free_percentage);
951 gclog_or_tty->print_cr(" Maximum free fraction %f",
952 maximum_free_percentage);
953 gclog_or_tty->print_cr(" Capactiy "SIZE_FORMAT, capacity()/1000);
954 gclog_or_tty->print_cr(" Desired capacity "SIZE_FORMAT,
955 desired_capacity/1000);
956 int prev_level = level() - 1;
957 if (prev_level >= 0) {
958 size_t prev_size = 0;
959 GenCollectedHeap* gch = GenCollectedHeap::heap();
960 Generation* prev_gen = gch->_gens[prev_level];
961 prev_size = prev_gen->capacity();
962 gclog_or_tty->print_cr(" Younger gen size "SIZE_FORMAT,
963 prev_size/1000);
964 }
965 gclog_or_tty->print_cr(" unsafe_max_alloc_nogc "SIZE_FORMAT,
966 unsafe_max_alloc_nogc()/1000);
967 gclog_or_tty->print_cr(" contiguous available "SIZE_FORMAT,
968 contiguous_available()/1000);
969 gclog_or_tty->print_cr(" Expand by "SIZE_FORMAT" (bytes)",
970 expand_bytes);
971 }
972 // safe if expansion fails
973 expand(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
974 if (PrintGCDetails && Verbose) {
975 gclog_or_tty->print_cr(" Expanded free fraction %f",
976 ((double) free()) / capacity());
977 }
978 }
979 }
981 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
982 return cmsSpace()->freelistLock();
983 }
985 HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size,
986 bool tlab) {
987 CMSSynchronousYieldRequest yr;
988 MutexLockerEx x(freelistLock(),
989 Mutex::_no_safepoint_check_flag);
990 return have_lock_and_allocate(size, tlab);
991 }
993 HeapWord* ConcurrentMarkSweepGeneration::have_lock_and_allocate(size_t size,
994 bool tlab) {
995 assert_lock_strong(freelistLock());
996 size_t adjustedSize = CompactibleFreeListSpace::adjustObjectSize(size);
997 HeapWord* res = cmsSpace()->allocate(adjustedSize);
998 // Allocate the object live (grey) if the background collector has
999 // started marking. This is necessary because the marker may
1000 // have passed this address and consequently this object will
1001 // not otherwise be greyed and would be incorrectly swept up.
1002 // Note that if this object contains references, the writing
1003 // of those references will dirty the card containing this object
1004 // allowing the object to be blackened (and its references scanned)
1005 // either during a preclean phase or at the final checkpoint.
1006 if (res != NULL) {
1007 collector()->direct_allocated(res, adjustedSize);
1008 _direct_allocated_words += adjustedSize;
1009 // allocation counters
1010 NOT_PRODUCT(
1011 _numObjectsAllocated++;
1012 _numWordsAllocated += (int)adjustedSize;
1013 )
1014 }
1015 return res;
1016 }
1018 // In the case of direct allocation by mutators in a generation that
1019 // is being concurrently collected, the object must be allocated
1020 // live (grey) if the background collector has started marking.
1021 // This is necessary because the marker may
1022 // have passed this address and consequently this object will
1023 // not otherwise be greyed and would be incorrectly swept up.
1024 // Note that if this object contains references, the writing
1025 // of those references will dirty the card containing this object
1026 // allowing the object to be blackened (and its references scanned)
1027 // either during a preclean phase or at the final checkpoint.
1028 void CMSCollector::direct_allocated(HeapWord* start, size_t size) {
1029 assert(_markBitMap.covers(start, size), "Out of bounds");
1030 if (_collectorState >= Marking) {
1031 MutexLockerEx y(_markBitMap.lock(),
1032 Mutex::_no_safepoint_check_flag);
1033 // [see comments preceding SweepClosure::do_blk() below for details]
1034 // 1. need to mark the object as live so it isn't collected
1035 // 2. need to mark the 2nd bit to indicate the object may be uninitialized
1036 // 3. need to mark the end of the object so sweeper can skip over it
1037 // if it's uninitialized when the sweeper reaches it.
1038 _markBitMap.mark(start); // object is live
1039 _markBitMap.mark(start + 1); // object is potentially uninitialized?
1040 _markBitMap.mark(start + size - 1);
1041 // mark end of object
1042 }
1043 // check that oop looks uninitialized
1044 assert(oop(start)->klass() == NULL, "_klass should be NULL");
1045 }
1047 void CMSCollector::promoted(bool par, HeapWord* start,
1048 bool is_obj_array, size_t obj_size) {
1049 assert(_markBitMap.covers(start), "Out of bounds");
1050 // See comment in direct_allocated() about when objects should
1051 // be allocated live.
1052 if (_collectorState >= Marking) {
1053 // we already hold the marking bit map lock, taken in
1054 // the prologue
1055 if (par) {
1056 _markBitMap.par_mark(start);
1057 } else {
1058 _markBitMap.mark(start);
1059 }
1060 // We don't need to mark the object as uninitialized (as
1061 // in direct_allocated above) because this is being done with the
1062 // world stopped and the object will be initialized by the
1063 // time the sweeper gets to look at it.
1064 assert(SafepointSynchronize::is_at_safepoint(),
1065 "expect promotion only at safepoints");
1067 if (_collectorState < Sweeping) {
1068 // Mark the appropriate cards in the modUnionTable, so that
1069 // this object gets scanned before the sweep. If this is
1070 // not done, CMS generation references in the object might
1071 // not get marked.
1072 // For the case of arrays, which are otherwise precisely
1073 // marked, we need to dirty the entire array, not just its head.
1074 if (is_obj_array) {
1075 // The [par_]mark_range() method expects mr.end() below to
1076 // be aligned to the granularity of a bit's representation
1077 // in the heap. In the case of the MUT below, that's a
1078 // card size.
1079 MemRegion mr(start,
1080 (HeapWord*)round_to((intptr_t)(start + obj_size),
1081 CardTableModRefBS::card_size /* bytes */));
1082 if (par) {
1083 _modUnionTable.par_mark_range(mr);
1084 } else {
1085 _modUnionTable.mark_range(mr);
1086 }
1087 } else { // not an obj array; we can just mark the head
1088 if (par) {
1089 _modUnionTable.par_mark(start);
1090 } else {
1091 _modUnionTable.mark(start);
1092 }
1093 }
1094 }
1095 }
1096 }
1098 static inline size_t percent_of_space(Space* space, HeapWord* addr)
1099 {
1100 size_t delta = pointer_delta(addr, space->bottom());
1101 return (size_t)(delta * 100.0 / (space->capacity() / HeapWordSize));
1102 }
1104 void CMSCollector::icms_update_allocation_limits()
1105 {
1106 Generation* gen0 = GenCollectedHeap::heap()->get_gen(0);
1107 EdenSpace* eden = gen0->as_DefNewGeneration()->eden();
1109 const unsigned int duty_cycle = stats().icms_update_duty_cycle();
1110 if (CMSTraceIncrementalPacing) {
1111 stats().print();
1112 }
1114 assert(duty_cycle <= 100, "invalid duty cycle");
1115 if (duty_cycle != 0) {
1116 // The duty_cycle is a percentage between 0 and 100; convert to words and
1117 // then compute the offset from the endpoints of the space.
1118 size_t free_words = eden->free() / HeapWordSize;
1119 double free_words_dbl = (double)free_words;
1120 size_t duty_cycle_words = (size_t)(free_words_dbl * duty_cycle / 100.0);
1121 size_t offset_words = (free_words - duty_cycle_words) / 2;
1123 _icms_start_limit = eden->top() + offset_words;
1124 _icms_stop_limit = eden->end() - offset_words;
1126 // The limits may be adjusted (shifted to the right) by
1127 // CMSIncrementalOffset, to allow the application more mutator time after a
1128 // young gen gc (when all mutators were stopped) and before CMS starts and
1129 // takes away one or more cpus.
1130 if (CMSIncrementalOffset != 0) {
1131 double adjustment_dbl = free_words_dbl * CMSIncrementalOffset / 100.0;
1132 size_t adjustment = (size_t)adjustment_dbl;
1133 HeapWord* tmp_stop = _icms_stop_limit + adjustment;
1134 if (tmp_stop > _icms_stop_limit && tmp_stop < eden->end()) {
1135 _icms_start_limit += adjustment;
1136 _icms_stop_limit = tmp_stop;
1137 }
1138 }
1139 }
1140 if (duty_cycle == 0 || (_icms_start_limit == _icms_stop_limit)) {
1141 _icms_start_limit = _icms_stop_limit = eden->end();
1142 }
1144 // Install the new start limit.
1145 eden->set_soft_end(_icms_start_limit);
1147 if (CMSTraceIncrementalMode) {
1148 gclog_or_tty->print(" icms alloc limits: "
1149 PTR_FORMAT "," PTR_FORMAT
1150 " (" SIZE_FORMAT "%%," SIZE_FORMAT "%%) ",
1151 _icms_start_limit, _icms_stop_limit,
1152 percent_of_space(eden, _icms_start_limit),
1153 percent_of_space(eden, _icms_stop_limit));
1154 if (Verbose) {
1155 gclog_or_tty->print("eden: ");
1156 eden->print_on(gclog_or_tty);
1157 }
1158 }
1159 }
1161 // Any changes here should try to maintain the invariant
1162 // that if this method is called with _icms_start_limit
1163 // and _icms_stop_limit both NULL, then it should return NULL
1164 // and not notify the icms thread.
1165 HeapWord*
1166 CMSCollector::allocation_limit_reached(Space* space, HeapWord* top,
1167 size_t word_size)
1168 {
1169 // A start_limit equal to end() means the duty cycle is 0, so treat that as a
1170 // nop.
1171 if (CMSIncrementalMode && _icms_start_limit != space->end()) {
1172 if (top <= _icms_start_limit) {
1173 if (CMSTraceIncrementalMode) {
1174 space->print_on(gclog_or_tty);
1175 gclog_or_tty->stamp();
1176 gclog_or_tty->print_cr(" start limit top=" PTR_FORMAT
1177 ", new limit=" PTR_FORMAT
1178 " (" SIZE_FORMAT "%%)",
1179 top, _icms_stop_limit,
1180 percent_of_space(space, _icms_stop_limit));
1181 }
1182 ConcurrentMarkSweepThread::start_icms();
1183 assert(top < _icms_stop_limit, "Tautology");
1184 if (word_size < pointer_delta(_icms_stop_limit, top)) {
1185 return _icms_stop_limit;
1186 }
1188 // The allocation will cross both the _start and _stop limits, so do the
1189 // stop notification also and return end().
1190 if (CMSTraceIncrementalMode) {
1191 space->print_on(gclog_or_tty);
1192 gclog_or_tty->stamp();
1193 gclog_or_tty->print_cr(" +stop limit top=" PTR_FORMAT
1194 ", new limit=" PTR_FORMAT
1195 " (" SIZE_FORMAT "%%)",
1196 top, space->end(),
1197 percent_of_space(space, space->end()));
1198 }
1199 ConcurrentMarkSweepThread::stop_icms();
1200 return space->end();
1201 }
1203 if (top <= _icms_stop_limit) {
1204 if (CMSTraceIncrementalMode) {
1205 space->print_on(gclog_or_tty);
1206 gclog_or_tty->stamp();
1207 gclog_or_tty->print_cr(" stop limit top=" PTR_FORMAT
1208 ", new limit=" PTR_FORMAT
1209 " (" SIZE_FORMAT "%%)",
1210 top, space->end(),
1211 percent_of_space(space, space->end()));
1212 }
1213 ConcurrentMarkSweepThread::stop_icms();
1214 return space->end();
1215 }
1217 if (CMSTraceIncrementalMode) {
1218 space->print_on(gclog_or_tty);
1219 gclog_or_tty->stamp();
1220 gclog_or_tty->print_cr(" end limit top=" PTR_FORMAT
1221 ", new limit=" PTR_FORMAT,
1222 top, NULL);
1223 }
1224 }
1226 return NULL;
1227 }
1229 oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size, oop* ref) {
1230 assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");
1231 // allocate, copy and if necessary update promoinfo --
1232 // delegate to underlying space.
1233 assert_lock_strong(freelistLock());
1235 #ifndef PRODUCT
1236 if (Universe::heap()->promotion_should_fail()) {
1237 return NULL;
1238 }
1239 #endif // #ifndef PRODUCT
1241 oop res = _cmsSpace->promote(obj, obj_size, ref);
1242 if (res == NULL) {
1243 // expand and retry
1244 size_t s = _cmsSpace->expansionSpaceRequired(obj_size); // HeapWords
1245 expand(s*HeapWordSize, MinHeapDeltaBytes,
1246 CMSExpansionCause::_satisfy_promotion);
1247 // Since there's currently no next generation, we don't try to promote
1248 // into a more senior generation.
1249 assert(next_gen() == NULL, "assumption, based upon which no attempt "
1250 "is made to pass on a possibly failing "
1251 "promotion to next generation");
1252 res = _cmsSpace->promote(obj, obj_size, ref);
1253 }
1254 if (res != NULL) {
1255 // See comment in allocate() about when objects should
1256 // be allocated live.
1257 assert(obj->is_oop(), "Will dereference klass pointer below");
1258 collector()->promoted(false, // Not parallel
1259 (HeapWord*)res, obj->is_objArray(), obj_size);
1260 // promotion counters
1261 NOT_PRODUCT(
1262 _numObjectsPromoted++;
1263 _numWordsPromoted +=
1264 (int)(CompactibleFreeListSpace::adjustObjectSize(obj->size()));
1265 )
1266 }
1267 return res;
1268 }
1271 HeapWord*
1272 ConcurrentMarkSweepGeneration::allocation_limit_reached(Space* space,
1273 HeapWord* top,
1274 size_t word_sz)
1275 {
1276 return collector()->allocation_limit_reached(space, top, word_sz);
1277 }
1279 // Things to support parallel young-gen collection.
1280 oop
1281 ConcurrentMarkSweepGeneration::par_promote(int thread_num,
1282 oop old, markOop m,
1283 size_t word_sz) {
1284 #ifndef PRODUCT
1285 if (Universe::heap()->promotion_should_fail()) {
1286 return NULL;
1287 }
1288 #endif // #ifndef PRODUCT
1290 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1291 PromotionInfo* promoInfo = &ps->promo;
1292 // if we are tracking promotions, then first ensure space for
1293 // promotion (including spooling space for saving header if necessary).
1294 // then allocate and copy, then track promoted info if needed.
1295 // When tracking (see PromotionInfo::track()), the mark word may
1296 // be displaced and in this case restoration of the mark word
1297 // occurs in the (oop_since_save_marks_)iterate phase.
1298 if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {
1299 // Out of space for allocating spooling buffers;
1300 // try expanding and allocating spooling buffers.
1301 if (!expand_and_ensure_spooling_space(promoInfo)) {
1302 return NULL;
1303 }
1304 }
1305 assert(promoInfo->has_spooling_space(), "Control point invariant");
1306 HeapWord* obj_ptr = ps->lab.alloc(word_sz);
1307 if (obj_ptr == NULL) {
1308 obj_ptr = expand_and_par_lab_allocate(ps, word_sz);
1309 if (obj_ptr == NULL) {
1310 return NULL;
1311 }
1312 }
1313 oop obj = oop(obj_ptr);
1314 assert(obj->klass() == NULL, "Object should be uninitialized here.");
1315 // Otherwise, copy the object. Here we must be careful to insert the
1316 // klass pointer last, since this marks the block as an allocated object.
1317 HeapWord* old_ptr = (HeapWord*)old;
1318 if (word_sz > (size_t)oopDesc::header_size()) {
1319 Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),
1320 obj_ptr + oopDesc::header_size(),
1321 word_sz - oopDesc::header_size());
1322 }
1323 // Restore the mark word copied above.
1324 obj->set_mark(m);
1325 // Now we can track the promoted object, if necessary. We take care
1326 // To delay the transition from uninitialized to full object
1327 // (i.e., insertion of klass pointer) until after, so that it
1328 // atomically becomes a promoted object.
1329 if (promoInfo->tracking()) {
1330 promoInfo->track((PromotedObject*)obj, old->klass());
1331 }
1332 // Finally, install the klass pointer.
1333 obj->set_klass(old->klass());
1335 assert(old->is_oop(), "Will dereference klass ptr below");
1336 collector()->promoted(true, // parallel
1337 obj_ptr, old->is_objArray(), word_sz);
1339 NOT_PRODUCT(
1340 Atomic::inc(&_numObjectsPromoted);
1341 Atomic::add((jint)CompactibleFreeListSpace::adjustObjectSize(obj->size()),
1342 &_numWordsPromoted);
1343 )
1345 return obj;
1346 }
1348 void
1349 ConcurrentMarkSweepGeneration::
1350 par_promote_alloc_undo(int thread_num,
1351 HeapWord* obj, size_t word_sz) {
1352 // CMS does not support promotion undo.
1353 ShouldNotReachHere();
1354 }
1356 void
1357 ConcurrentMarkSweepGeneration::
1358 par_promote_alloc_done(int thread_num) {
1359 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1360 ps->lab.retire();
1361 #if CFLS_LAB_REFILL_STATS
1362 if (thread_num == 0) {
1363 _cmsSpace->print_par_alloc_stats();
1364 }
1365 #endif
1366 }
1368 void
1369 ConcurrentMarkSweepGeneration::
1370 par_oop_since_save_marks_iterate_done(int thread_num) {
1371 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1372 ParScanWithoutBarrierClosure* dummy_cl = NULL;
1373 ps->promo.promoted_oops_iterate_nv(dummy_cl);
1374 }
1376 // XXXPERM
1377 bool ConcurrentMarkSweepGeneration::should_collect(bool full,
1378 size_t size,
1379 bool tlab)
1380 {
1381 // We allow a STW collection only if a full
1382 // collection was requested.
1383 return full || should_allocate(size, tlab); // FIX ME !!!
1384 // This and promotion failure handling are connected at the
1385 // hip and should be fixed by untying them.
1386 }
1388 bool CMSCollector::shouldConcurrentCollect() {
1389 if (_full_gc_requested) {
1390 assert(ExplicitGCInvokesConcurrent, "Unexpected state");
1391 if (Verbose && PrintGCDetails) {
1392 gclog_or_tty->print_cr("CMSCollector: collect because of explicit "
1393 " gc request");
1394 }
1395 return true;
1396 }
1398 // For debugging purposes, change the type of collection.
1399 // If the rotation is not on the concurrent collection
1400 // type, don't start a concurrent collection.
1401 NOT_PRODUCT(
1402 if (RotateCMSCollectionTypes &&
1403 (_cmsGen->debug_collection_type() !=
1404 ConcurrentMarkSweepGeneration::Concurrent_collection_type)) {
1405 assert(_cmsGen->debug_collection_type() !=
1406 ConcurrentMarkSweepGeneration::Unknown_collection_type,
1407 "Bad cms collection type");
1408 return false;
1409 }
1410 )
1412 FreelistLocker x(this);
1413 // ------------------------------------------------------------------
1414 // Print out lots of information which affects the initiation of
1415 // a collection.
1416 if (PrintCMSInitiationStatistics && stats().valid()) {
1417 gclog_or_tty->print("CMSCollector shouldConcurrentCollect: ");
1418 gclog_or_tty->stamp();
1419 gclog_or_tty->print_cr("");
1420 stats().print_on(gclog_or_tty);
1421 gclog_or_tty->print_cr("time_until_cms_gen_full %3.7f",
1422 stats().time_until_cms_gen_full());
1423 gclog_or_tty->print_cr("free="SIZE_FORMAT, _cmsGen->free());
1424 gclog_or_tty->print_cr("contiguous_available="SIZE_FORMAT,
1425 _cmsGen->contiguous_available());
1426 gclog_or_tty->print_cr("promotion_rate=%g", stats().promotion_rate());
1427 gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate());
1428 gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy());
1429 gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy());
1430 gclog_or_tty->print_cr("initiatingPermOccupancy=%3.7f", _permGen->initiating_occupancy());
1431 }
1432 // ------------------------------------------------------------------
1434 // If the estimated time to complete a cms collection (cms_duration())
1435 // is less than the estimated time remaining until the cms generation
1436 // is full, start a collection.
1437 if (!UseCMSInitiatingOccupancyOnly) {
1438 if (stats().valid()) {
1439 if (stats().time_until_cms_start() == 0.0) {
1440 return true;
1441 }
1442 } else {
1443 // We want to conservatively collect somewhat early in order
1444 // to try and "bootstrap" our CMS/promotion statistics;
1445 // this branch will not fire after the first successful CMS
1446 // collection because the stats should then be valid.
1447 if (_cmsGen->occupancy() >= _bootstrap_occupancy) {
1448 if (Verbose && PrintGCDetails) {
1449 gclog_or_tty->print_cr(
1450 " CMSCollector: collect for bootstrapping statistics:"
1451 " occupancy = %f, boot occupancy = %f", _cmsGen->occupancy(),
1452 _bootstrap_occupancy);
1453 }
1454 return true;
1455 }
1456 }
1457 }
1459 // Otherwise, we start a collection cycle if either the perm gen or
1460 // old gen want a collection cycle started. Each may use
1461 // an appropriate criterion for making this decision.
1462 // XXX We need to make sure that the gen expansion
1463 // criterion dovetails well with this. XXX NEED TO FIX THIS
1464 if (_cmsGen->should_concurrent_collect()) {
1465 if (Verbose && PrintGCDetails) {
1466 gclog_or_tty->print_cr("CMS old gen initiated");
1467 }
1468 return true;
1469 }
1471 // We start a collection if we believe an incremental collection may fail;
1472 // this is not likely to be productive in practice because it's probably too
1473 // late anyway.
1474 GenCollectedHeap* gch = GenCollectedHeap::heap();
1475 assert(gch->collector_policy()->is_two_generation_policy(),
1476 "You may want to check the correctness of the following");
1477 if (gch->incremental_collection_will_fail()) {
1478 if (PrintGCDetails && Verbose) {
1479 gclog_or_tty->print("CMSCollector: collect because incremental collection will fail ");
1480 }
1481 return true;
1482 }
1484 if (CMSClassUnloadingEnabled && _permGen->should_concurrent_collect()) {
1485 bool res = update_should_unload_classes();
1486 if (res) {
1487 if (Verbose && PrintGCDetails) {
1488 gclog_or_tty->print_cr("CMS perm gen initiated");
1489 }
1490 return true;
1491 }
1492 }
1493 return false;
1494 }
1496 // Clear _expansion_cause fields of constituent generations
1497 void CMSCollector::clear_expansion_cause() {
1498 _cmsGen->clear_expansion_cause();
1499 _permGen->clear_expansion_cause();
1500 }
1502 // We should be conservative in starting a collection cycle. To
1503 // start too eagerly runs the risk of collecting too often in the
1504 // extreme. To collect too rarely falls back on full collections,
1505 // which works, even if not optimum in terms of concurrent work.
1506 // As a work around for too eagerly collecting, use the flag
1507 // UseCMSInitiatingOccupancyOnly. This also has the advantage of
1508 // giving the user an easily understandable way of controlling the
1509 // collections.
1510 // We want to start a new collection cycle if any of the following
1511 // conditions hold:
1512 // . our current occupancy exceeds the configured initiating occupancy
1513 // for this generation, or
1514 // . we recently needed to expand this space and have not, since that
1515 // expansion, done a collection of this generation, or
1516 // . the underlying space believes that it may be a good idea to initiate
1517 // a concurrent collection (this may be based on criteria such as the
1518 // following: the space uses linear allocation and linear allocation is
1519 // going to fail, or there is believed to be excessive fragmentation in
1520 // the generation, etc... or ...
1521 // [.(currently done by CMSCollector::shouldConcurrentCollect() only for
1522 // the case of the old generation, not the perm generation; see CR 6543076):
1523 // we may be approaching a point at which allocation requests may fail because
1524 // we will be out of sufficient free space given allocation rate estimates.]
1525 bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const {
1527 assert_lock_strong(freelistLock());
1528 if (occupancy() > initiating_occupancy()) {
1529 if (PrintGCDetails && Verbose) {
1530 gclog_or_tty->print(" %s: collect because of occupancy %f / %f ",
1531 short_name(), occupancy(), initiating_occupancy());
1532 }
1533 return true;
1534 }
1535 if (UseCMSInitiatingOccupancyOnly) {
1536 return false;
1537 }
1538 if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) {
1539 if (PrintGCDetails && Verbose) {
1540 gclog_or_tty->print(" %s: collect because expanded for allocation ",
1541 short_name());
1542 }
1543 return true;
1544 }
1545 if (_cmsSpace->should_concurrent_collect()) {
1546 if (PrintGCDetails && Verbose) {
1547 gclog_or_tty->print(" %s: collect because cmsSpace says so ",
1548 short_name());
1549 }
1550 return true;
1551 }
1552 return false;
1553 }
1555 void ConcurrentMarkSweepGeneration::collect(bool full,
1556 bool clear_all_soft_refs,
1557 size_t size,
1558 bool tlab)
1559 {
1560 collector()->collect(full, clear_all_soft_refs, size, tlab);
1561 }
1563 void CMSCollector::collect(bool full,
1564 bool clear_all_soft_refs,
1565 size_t size,
1566 bool tlab)
1567 {
1568 if (!UseCMSCollectionPassing && _collectorState > Idling) {
1569 // For debugging purposes skip the collection if the state
1570 // is not currently idle
1571 if (TraceCMSState) {
1572 gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " skipped full:%d CMS state %d",
1573 Thread::current(), full, _collectorState);
1574 }
1575 return;
1576 }
1578 // The following "if" branch is present for defensive reasons.
1579 // In the current uses of this interface, it can be replaced with:
1580 // assert(!GC_locker.is_active(), "Can't be called otherwise");
1581 // But I am not placing that assert here to allow future
1582 // generality in invoking this interface.
1583 if (GC_locker::is_active()) {
1584 // A consistency test for GC_locker
1585 assert(GC_locker::needs_gc(), "Should have been set already");
1586 // Skip this foreground collection, instead
1587 // expanding the heap if necessary.
1588 // Need the free list locks for the call to free() in compute_new_size()
1589 compute_new_size();
1590 return;
1591 }
1592 acquire_control_and_collect(full, clear_all_soft_refs);
1593 _full_gcs_since_conc_gc++;
1595 }
1597 void CMSCollector::request_full_gc(unsigned int full_gc_count) {
1598 GenCollectedHeap* gch = GenCollectedHeap::heap();
1599 unsigned int gc_count = gch->total_full_collections();
1600 if (gc_count == full_gc_count) {
1601 MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag);
1602 _full_gc_requested = true;
1603 CGC_lock->notify(); // nudge CMS thread
1604 }
1605 }
1608 // The foreground and background collectors need to coordinate in order
1609 // to make sure that they do not mutually interfere with CMS collections.
1610 // When a background collection is active,
1611 // the foreground collector may need to take over (preempt) and
1612 // synchronously complete an ongoing collection. Depending on the
1613 // frequency of the background collections and the heap usage
1614 // of the application, this preemption can be seldom or frequent.
1615 // There are only certain
1616 // points in the background collection that the "collection-baton"
1617 // can be passed to the foreground collector.
1618 //
1619 // The foreground collector will wait for the baton before
1620 // starting any part of the collection. The foreground collector
1621 // will only wait at one location.
1622 //
1623 // The background collector will yield the baton before starting a new
1624 // phase of the collection (e.g., before initial marking, marking from roots,
1625 // precleaning, final re-mark, sweep etc.) This is normally done at the head
1626 // of the loop which switches the phases. The background collector does some
1627 // of the phases (initial mark, final re-mark) with the world stopped.
1628 // Because of locking involved in stopping the world,
1629 // the foreground collector should not block waiting for the background
1630 // collector when it is doing a stop-the-world phase. The background
1631 // collector will yield the baton at an additional point just before
1632 // it enters a stop-the-world phase. Once the world is stopped, the
1633 // background collector checks the phase of the collection. If the
1634 // phase has not changed, it proceeds with the collection. If the
1635 // phase has changed, it skips that phase of the collection. See
1636 // the comments on the use of the Heap_lock in collect_in_background().
1637 //
1638 // Variable used in baton passing.
1639 // _foregroundGCIsActive - Set to true by the foreground collector when
1640 // it wants the baton. The foreground clears it when it has finished
1641 // the collection.
1642 // _foregroundGCShouldWait - Set to true by the background collector
1643 // when it is running. The foreground collector waits while
1644 // _foregroundGCShouldWait is true.
1645 // CGC_lock - monitor used to protect access to the above variables
1646 // and to notify the foreground and background collectors.
1647 // _collectorState - current state of the CMS collection.
1648 //
1649 // The foreground collector
1650 // acquires the CGC_lock
1651 // sets _foregroundGCIsActive
1652 // waits on the CGC_lock for _foregroundGCShouldWait to be false
1653 // various locks acquired in preparation for the collection
1654 // are released so as not to block the background collector
1655 // that is in the midst of a collection
1656 // proceeds with the collection
1657 // clears _foregroundGCIsActive
1658 // returns
1659 //
1660 // The background collector in a loop iterating on the phases of the
1661 // collection
1662 // acquires the CGC_lock
1663 // sets _foregroundGCShouldWait
1664 // if _foregroundGCIsActive is set
1665 // clears _foregroundGCShouldWait, notifies _CGC_lock
1666 // waits on _CGC_lock for _foregroundGCIsActive to become false
1667 // and exits the loop.
1668 // otherwise
1669 // proceed with that phase of the collection
1670 // if the phase is a stop-the-world phase,
1671 // yield the baton once more just before enqueueing
1672 // the stop-world CMS operation (executed by the VM thread).
1673 // returns after all phases of the collection are done
1674 //
1676 void CMSCollector::acquire_control_and_collect(bool full,
1677 bool clear_all_soft_refs) {
1678 assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
1679 assert(!Thread::current()->is_ConcurrentGC_thread(),
1680 "shouldn't try to acquire control from self!");
1682 // Start the protocol for acquiring control of the
1683 // collection from the background collector (aka CMS thread).
1684 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1685 "VM thread should have CMS token");
1686 // Remember the possibly interrupted state of an ongoing
1687 // concurrent collection
1688 CollectorState first_state = _collectorState;
1690 // Signal to a possibly ongoing concurrent collection that
1691 // we want to do a foreground collection.
1692 _foregroundGCIsActive = true;
1694 // Disable incremental mode during a foreground collection.
1695 ICMSDisabler icms_disabler;
1697 // release locks and wait for a notify from the background collector
1698 // releasing the locks in only necessary for phases which
1699 // do yields to improve the granularity of the collection.
1700 assert_lock_strong(bitMapLock());
1701 // We need to lock the Free list lock for the space that we are
1702 // currently collecting.
1703 assert(haveFreelistLocks(), "Must be holding free list locks");
1704 bitMapLock()->unlock();
1705 releaseFreelistLocks();
1706 {
1707 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1708 if (_foregroundGCShouldWait) {
1709 // We are going to be waiting for action for the CMS thread;
1710 // it had better not be gone (for instance at shutdown)!
1711 assert(ConcurrentMarkSweepThread::cmst() != NULL,
1712 "CMS thread must be running");
1713 // Wait here until the background collector gives us the go-ahead
1714 ConcurrentMarkSweepThread::clear_CMS_flag(
1715 ConcurrentMarkSweepThread::CMS_vm_has_token); // release token
1716 // Get a possibly blocked CMS thread going:
1717 // Note that we set _foregroundGCIsActive true above,
1718 // without protection of the CGC_lock.
1719 CGC_lock->notify();
1720 assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(),
1721 "Possible deadlock");
1722 while (_foregroundGCShouldWait) {
1723 // wait for notification
1724 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1725 // Possibility of delay/starvation here, since CMS token does
1726 // not know to give priority to VM thread? Actually, i think
1727 // there wouldn't be any delay/starvation, but the proof of
1728 // that "fact" (?) appears non-trivial. XXX 20011219YSR
1729 }
1730 ConcurrentMarkSweepThread::set_CMS_flag(
1731 ConcurrentMarkSweepThread::CMS_vm_has_token);
1732 }
1733 }
1734 // The CMS_token is already held. Get back the other locks.
1735 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1736 "VM thread should have CMS token");
1737 getFreelistLocks();
1738 bitMapLock()->lock_without_safepoint_check();
1739 if (TraceCMSState) {
1740 gclog_or_tty->print_cr("CMS foreground collector has asked for control "
1741 INTPTR_FORMAT " with first state %d", Thread::current(), first_state);
1742 gclog_or_tty->print_cr(" gets control with state %d", _collectorState);
1743 }
1745 // Check if we need to do a compaction, or if not, whether
1746 // we need to start the mark-sweep from scratch.
1747 bool should_compact = false;
1748 bool should_start_over = false;
1749 decide_foreground_collection_type(clear_all_soft_refs,
1750 &should_compact, &should_start_over);
1752 NOT_PRODUCT(
1753 if (RotateCMSCollectionTypes) {
1754 if (_cmsGen->debug_collection_type() ==
1755 ConcurrentMarkSweepGeneration::MSC_foreground_collection_type) {
1756 should_compact = true;
1757 } else if (_cmsGen->debug_collection_type() ==
1758 ConcurrentMarkSweepGeneration::MS_foreground_collection_type) {
1759 should_compact = false;
1760 }
1761 }
1762 )
1764 if (PrintGCDetails && first_state > Idling) {
1765 GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause();
1766 if (GCCause::is_user_requested_gc(cause) ||
1767 GCCause::is_serviceability_requested_gc(cause)) {
1768 gclog_or_tty->print(" (concurrent mode interrupted)");
1769 } else {
1770 gclog_or_tty->print(" (concurrent mode failure)");
1771 }
1772 }
1774 if (should_compact) {
1775 // If the collection is being acquired from the background
1776 // collector, there may be references on the discovered
1777 // references lists that have NULL referents (being those
1778 // that were concurrently cleared by a mutator) or
1779 // that are no longer active (having been enqueued concurrently
1780 // by the mutator).
1781 // Scrub the list of those references because Mark-Sweep-Compact
1782 // code assumes referents are not NULL and that all discovered
1783 // Reference objects are active.
1784 ref_processor()->clean_up_discovered_references();
1786 do_compaction_work(clear_all_soft_refs);
1788 // Has the GC time limit been exceeded?
1789 check_gc_time_limit();
1791 } else {
1792 do_mark_sweep_work(clear_all_soft_refs, first_state,
1793 should_start_over);
1794 }
1795 // Reset the expansion cause, now that we just completed
1796 // a collection cycle.
1797 clear_expansion_cause();
1798 _foregroundGCIsActive = false;
1799 return;
1800 }
1802 void CMSCollector::check_gc_time_limit() {
1804 // Ignore explicit GC's. Exiting here does not set the flag and
1805 // does not reset the count. Updating of the averages for system
1806 // GC's is still controlled by UseAdaptiveSizePolicyWithSystemGC.
1807 GCCause::Cause gc_cause = GenCollectedHeap::heap()->gc_cause();
1808 if (GCCause::is_user_requested_gc(gc_cause) ||
1809 GCCause::is_serviceability_requested_gc(gc_cause)) {
1810 return;
1811 }
1813 // Calculate the fraction of the CMS generation was freed during
1814 // the last collection.
1815 // Only consider the STW compacting cost for now.
1816 //
1817 // Note that the gc time limit test only works for the collections
1818 // of the young gen + tenured gen and not for collections of the
1819 // permanent gen. That is because the calculation of the space
1820 // freed by the collection is the free space in the young gen +
1821 // tenured gen.
1823 double fraction_free =
1824 ((double)_cmsGen->free())/((double)_cmsGen->max_capacity());
1825 if ((100.0 * size_policy()->compacting_gc_cost()) >
1826 ((double) GCTimeLimit) &&
1827 ((fraction_free * 100) < GCHeapFreeLimit)) {
1828 size_policy()->inc_gc_time_limit_count();
1829 if (UseGCOverheadLimit &&
1830 (size_policy()->gc_time_limit_count() >
1831 AdaptiveSizePolicyGCTimeLimitThreshold)) {
1832 size_policy()->set_gc_time_limit_exceeded(true);
1833 // Avoid consecutive OOM due to the gc time limit by resetting
1834 // the counter.
1835 size_policy()->reset_gc_time_limit_count();
1836 if (PrintGCDetails) {
1837 gclog_or_tty->print_cr(" GC is exceeding overhead limit "
1838 "of %d%%", GCTimeLimit);
1839 }
1840 } else {
1841 if (PrintGCDetails) {
1842 gclog_or_tty->print_cr(" GC would exceed overhead limit "
1843 "of %d%%", GCTimeLimit);
1844 }
1845 }
1846 } else {
1847 size_policy()->reset_gc_time_limit_count();
1848 }
1849 }
1851 // Resize the perm generation and the tenured generation
1852 // after obtaining the free list locks for the
1853 // two generations.
1854 void CMSCollector::compute_new_size() {
1855 assert_locked_or_safepoint(Heap_lock);
1856 FreelistLocker z(this);
1857 _permGen->compute_new_size();
1858 _cmsGen->compute_new_size();
1859 }
1861 // A work method used by foreground collection to determine
1862 // what type of collection (compacting or not, continuing or fresh)
1863 // it should do.
1864 // NOTE: the intent is to make UseCMSCompactAtFullCollection
1865 // and CMSCompactWhenClearAllSoftRefs the default in the future
1866 // and do away with the flags after a suitable period.
1867 void CMSCollector::decide_foreground_collection_type(
1868 bool clear_all_soft_refs, bool* should_compact,
1869 bool* should_start_over) {
1870 // Normally, we'll compact only if the UseCMSCompactAtFullCollection
1871 // flag is set, and we have either requested a System.gc() or
1872 // the number of full gc's since the last concurrent cycle
1873 // has exceeded the threshold set by CMSFullGCsBeforeCompaction,
1874 // or if an incremental collection has failed
1875 GenCollectedHeap* gch = GenCollectedHeap::heap();
1876 assert(gch->collector_policy()->is_two_generation_policy(),
1877 "You may want to check the correctness of the following");
1878 // Inform cms gen if this was due to partial collection failing.
1879 // The CMS gen may use this fact to determine its expansion policy.
1880 if (gch->incremental_collection_will_fail()) {
1881 assert(!_cmsGen->incremental_collection_failed(),
1882 "Should have been noticed, reacted to and cleared");
1883 _cmsGen->set_incremental_collection_failed();
1884 }
1885 *should_compact =
1886 UseCMSCompactAtFullCollection &&
1887 ((_full_gcs_since_conc_gc >= CMSFullGCsBeforeCompaction) ||
1888 GCCause::is_user_requested_gc(gch->gc_cause()) ||
1889 gch->incremental_collection_will_fail());
1890 *should_start_over = false;
1891 if (clear_all_soft_refs && !*should_compact) {
1892 // We are about to do a last ditch collection attempt
1893 // so it would normally make sense to do a compaction
1894 // to reclaim as much space as possible.
1895 if (CMSCompactWhenClearAllSoftRefs) {
1896 // Default: The rationale is that in this case either
1897 // we are past the final marking phase, in which case
1898 // we'd have to start over, or so little has been done
1899 // that there's little point in saving that work. Compaction
1900 // appears to be the sensible choice in either case.
1901 *should_compact = true;
1902 } else {
1903 // We have been asked to clear all soft refs, but not to
1904 // compact. Make sure that we aren't past the final checkpoint
1905 // phase, for that is where we process soft refs. If we are already
1906 // past that phase, we'll need to redo the refs discovery phase and
1907 // if necessary clear soft refs that weren't previously
1908 // cleared. We do so by remembering the phase in which
1909 // we came in, and if we are past the refs processing
1910 // phase, we'll choose to just redo the mark-sweep
1911 // collection from scratch.
1912 if (_collectorState > FinalMarking) {
1913 // We are past the refs processing phase;
1914 // start over and do a fresh synchronous CMS cycle
1915 _collectorState = Resetting; // skip to reset to start new cycle
1916 reset(false /* == !asynch */);
1917 *should_start_over = true;
1918 } // else we can continue a possibly ongoing current cycle
1919 }
1920 }
1921 }
1923 // A work method used by the foreground collector to do
1924 // a mark-sweep-compact.
1925 void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
1926 GenCollectedHeap* gch = GenCollectedHeap::heap();
1927 TraceTime t("CMS:MSC ", PrintGCDetails && Verbose, true, gclog_or_tty);
1928 if (PrintGC && Verbose && !(GCCause::is_user_requested_gc(gch->gc_cause()))) {
1929 gclog_or_tty->print_cr("Compact ConcurrentMarkSweepGeneration after %d "
1930 "collections passed to foreground collector", _full_gcs_since_conc_gc);
1931 }
1933 // Sample collection interval time and reset for collection pause.
1934 if (UseAdaptiveSizePolicy) {
1935 size_policy()->msc_collection_begin();
1936 }
1938 // Temporarily widen the span of the weak reference processing to
1939 // the entire heap.
1940 MemRegion new_span(GenCollectedHeap::heap()->reserved_region());
1941 ReferenceProcessorSpanMutator x(ref_processor(), new_span);
1943 // Temporarily, clear the "is_alive_non_header" field of the
1944 // reference processor.
1945 ReferenceProcessorIsAliveMutator y(ref_processor(), NULL);
1947 // Temporarily make reference _processing_ single threaded (non-MT).
1948 ReferenceProcessorMTProcMutator z(ref_processor(), false);
1950 // Temporarily make refs discovery atomic
1951 ReferenceProcessorAtomicMutator w(ref_processor(), true);
1953 ref_processor()->set_enqueuing_is_done(false);
1954 ref_processor()->enable_discovery();
1955 // If an asynchronous collection finishes, the _modUnionTable is
1956 // all clear. If we are assuming the collection from an asynchronous
1957 // collection, clear the _modUnionTable.
1958 assert(_collectorState != Idling || _modUnionTable.isAllClear(),
1959 "_modUnionTable should be clear if the baton was not passed");
1960 _modUnionTable.clear_all();
1962 // We must adjust the allocation statistics being maintained
1963 // in the free list space. We do so by reading and clearing
1964 // the sweep timer and updating the block flux rate estimates below.
1965 assert(_sweep_timer.is_active(), "We should never see the timer inactive");
1966 _sweep_timer.stop();
1967 // Note that we do not use this sample to update the _sweep_estimate.
1968 _cmsGen->cmsSpace()->beginSweepFLCensus((float)(_sweep_timer.seconds()),
1969 _sweep_estimate.padded_average());
1971 GenMarkSweep::invoke_at_safepoint(_cmsGen->level(),
1972 ref_processor(), clear_all_soft_refs);
1973 #ifdef ASSERT
1974 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
1975 size_t free_size = cms_space->free();
1976 assert(free_size ==
1977 pointer_delta(cms_space->end(), cms_space->compaction_top())
1978 * HeapWordSize,
1979 "All the free space should be compacted into one chunk at top");
1980 assert(cms_space->dictionary()->totalChunkSize(
1981 debug_only(cms_space->freelistLock())) == 0 ||
1982 cms_space->totalSizeInIndexedFreeLists() == 0,
1983 "All the free space should be in a single chunk");
1984 size_t num = cms_space->totalCount();
1985 assert((free_size == 0 && num == 0) ||
1986 (free_size > 0 && (num == 1 || num == 2)),
1987 "There should be at most 2 free chunks after compaction");
1988 #endif // ASSERT
1989 _collectorState = Resetting;
1990 assert(_restart_addr == NULL,
1991 "Should have been NULL'd before baton was passed");
1992 reset(false /* == !asynch */);
1993 _cmsGen->reset_after_compaction();
1994 _concurrent_cycles_since_last_unload = 0;
1996 if (verifying() && !should_unload_classes()) {
1997 perm_gen_verify_bit_map()->clear_all();
1998 }
2000 // Clear any data recorded in the PLAB chunk arrays.
2001 if (_survivor_plab_array != NULL) {
2002 reset_survivor_plab_arrays();
2003 }
2005 // Adjust the per-size allocation stats for the next epoch.
2006 _cmsGen->cmsSpace()->endSweepFLCensus(sweepCount() /* fake */);
2007 // Restart the "sweep timer" for next epoch.
2008 _sweep_timer.reset();
2009 _sweep_timer.start();
2011 // Sample collection pause time and reset for collection interval.
2012 if (UseAdaptiveSizePolicy) {
2013 size_policy()->msc_collection_end(gch->gc_cause());
2014 }
2016 // For a mark-sweep-compact, compute_new_size() will be called
2017 // in the heap's do_collection() method.
2018 }
2020 // A work method used by the foreground collector to do
2021 // a mark-sweep, after taking over from a possibly on-going
2022 // concurrent mark-sweep collection.
2023 void CMSCollector::do_mark_sweep_work(bool clear_all_soft_refs,
2024 CollectorState first_state, bool should_start_over) {
2025 if (PrintGC && Verbose) {
2026 gclog_or_tty->print_cr("Pass concurrent collection to foreground "
2027 "collector with count %d",
2028 _full_gcs_since_conc_gc);
2029 }
2030 switch (_collectorState) {
2031 case Idling:
2032 if (first_state == Idling || should_start_over) {
2033 // The background GC was not active, or should
2034 // restarted from scratch; start the cycle.
2035 _collectorState = InitialMarking;
2036 }
2037 // If first_state was not Idling, then a background GC
2038 // was in progress and has now finished. No need to do it
2039 // again. Leave the state as Idling.
2040 break;
2041 case Precleaning:
2042 // In the foreground case don't do the precleaning since
2043 // it is not done concurrently and there is extra work
2044 // required.
2045 _collectorState = FinalMarking;
2046 }
2047 if (PrintGCDetails &&
2048 (_collectorState > Idling ||
2049 !GCCause::is_user_requested_gc(GenCollectedHeap::heap()->gc_cause()))) {
2050 gclog_or_tty->print(" (concurrent mode failure)");
2051 }
2052 collect_in_foreground(clear_all_soft_refs);
2054 // For a mark-sweep, compute_new_size() will be called
2055 // in the heap's do_collection() method.
2056 }
2059 void CMSCollector::getFreelistLocks() const {
2060 // Get locks for all free lists in all generations that this
2061 // collector is responsible for
2062 _cmsGen->freelistLock()->lock_without_safepoint_check();
2063 _permGen->freelistLock()->lock_without_safepoint_check();
2064 }
2066 void CMSCollector::releaseFreelistLocks() const {
2067 // Release locks for all free lists in all generations that this
2068 // collector is responsible for
2069 _cmsGen->freelistLock()->unlock();
2070 _permGen->freelistLock()->unlock();
2071 }
2073 bool CMSCollector::haveFreelistLocks() const {
2074 // Check locks for all free lists in all generations that this
2075 // collector is responsible for
2076 assert_lock_strong(_cmsGen->freelistLock());
2077 assert_lock_strong(_permGen->freelistLock());
2078 PRODUCT_ONLY(ShouldNotReachHere());
2079 return true;
2080 }
2082 // A utility class that is used by the CMS collector to
2083 // temporarily "release" the foreground collector from its
2084 // usual obligation to wait for the background collector to
2085 // complete an ongoing phase before proceeding.
2086 class ReleaseForegroundGC: public StackObj {
2087 private:
2088 CMSCollector* _c;
2089 public:
2090 ReleaseForegroundGC(CMSCollector* c) : _c(c) {
2091 assert(_c->_foregroundGCShouldWait, "Else should not need to call");
2092 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2093 // allow a potentially blocked foreground collector to proceed
2094 _c->_foregroundGCShouldWait = false;
2095 if (_c->_foregroundGCIsActive) {
2096 CGC_lock->notify();
2097 }
2098 assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
2099 "Possible deadlock");
2100 }
2102 ~ReleaseForegroundGC() {
2103 assert(!_c->_foregroundGCShouldWait, "Usage protocol violation?");
2104 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2105 _c->_foregroundGCShouldWait = true;
2106 }
2107 };
2109 // There are separate collect_in_background and collect_in_foreground because of
2110 // the different locking requirements of the background collector and the
2111 // foreground collector. There was originally an attempt to share
2112 // one "collect" method between the background collector and the foreground
2113 // collector but the if-then-else required made it cleaner to have
2114 // separate methods.
2115 void CMSCollector::collect_in_background(bool clear_all_soft_refs) {
2116 assert(Thread::current()->is_ConcurrentGC_thread(),
2117 "A CMS asynchronous collection is only allowed on a CMS thread.");
2119 GenCollectedHeap* gch = GenCollectedHeap::heap();
2120 {
2121 bool safepoint_check = Mutex::_no_safepoint_check_flag;
2122 MutexLockerEx hl(Heap_lock, safepoint_check);
2123 FreelistLocker fll(this);
2124 MutexLockerEx x(CGC_lock, safepoint_check);
2125 if (_foregroundGCIsActive || !UseAsyncConcMarkSweepGC) {
2126 // The foreground collector is active or we're
2127 // not using asynchronous collections. Skip this
2128 // background collection.
2129 assert(!_foregroundGCShouldWait, "Should be clear");
2130 return;
2131 } else {
2132 assert(_collectorState == Idling, "Should be idling before start.");
2133 _collectorState = InitialMarking;
2134 // Reset the expansion cause, now that we are about to begin
2135 // a new cycle.
2136 clear_expansion_cause();
2137 }
2138 // Decide if we want to enable class unloading as part of the
2139 // ensuing concurrent GC cycle.
2140 update_should_unload_classes();
2141 _full_gc_requested = false; // acks all outstanding full gc requests
2142 // Signal that we are about to start a collection
2143 gch->increment_total_full_collections(); // ... starting a collection cycle
2144 _collection_count_start = gch->total_full_collections();
2145 }
2147 // Used for PrintGC
2148 size_t prev_used;
2149 if (PrintGC && Verbose) {
2150 prev_used = _cmsGen->used(); // XXXPERM
2151 }
2153 // The change of the collection state is normally done at this level;
2154 // the exceptions are phases that are executed while the world is
2155 // stopped. For those phases the change of state is done while the
2156 // world is stopped. For baton passing purposes this allows the
2157 // background collector to finish the phase and change state atomically.
2158 // The foreground collector cannot wait on a phase that is done
2159 // while the world is stopped because the foreground collector already
2160 // has the world stopped and would deadlock.
2161 while (_collectorState != Idling) {
2162 if (TraceCMSState) {
2163 gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
2164 Thread::current(), _collectorState);
2165 }
2166 // The foreground collector
2167 // holds the Heap_lock throughout its collection.
2168 // holds the CMS token (but not the lock)
2169 // except while it is waiting for the background collector to yield.
2170 //
2171 // The foreground collector should be blocked (not for long)
2172 // if the background collector is about to start a phase
2173 // executed with world stopped. If the background
2174 // collector has already started such a phase, the
2175 // foreground collector is blocked waiting for the
2176 // Heap_lock. The stop-world phases (InitialMarking and FinalMarking)
2177 // are executed in the VM thread.
2178 //
2179 // The locking order is
2180 // PendingListLock (PLL) -- if applicable (FinalMarking)
2181 // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue())
2182 // CMS token (claimed in
2183 // stop_world_and_do() -->
2184 // safepoint_synchronize() -->
2185 // CMSThread::synchronize())
2187 {
2188 // Check if the FG collector wants us to yield.
2189 CMSTokenSync x(true); // is cms thread
2190 if (waitForForegroundGC()) {
2191 // We yielded to a foreground GC, nothing more to be
2192 // done this round.
2193 assert(_foregroundGCShouldWait == false, "We set it to false in "
2194 "waitForForegroundGC()");
2195 if (TraceCMSState) {
2196 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
2197 " exiting collection CMS state %d",
2198 Thread::current(), _collectorState);
2199 }
2200 return;
2201 } else {
2202 // The background collector can run but check to see if the
2203 // foreground collector has done a collection while the
2204 // background collector was waiting to get the CGC_lock
2205 // above. If yes, break so that _foregroundGCShouldWait
2206 // is cleared before returning.
2207 if (_collectorState == Idling) {
2208 break;
2209 }
2210 }
2211 }
2213 assert(_foregroundGCShouldWait, "Foreground collector, if active, "
2214 "should be waiting");
2216 switch (_collectorState) {
2217 case InitialMarking:
2218 {
2219 ReleaseForegroundGC x(this);
2220 stats().record_cms_begin();
2222 VM_CMS_Initial_Mark initial_mark_op(this);
2223 VMThread::execute(&initial_mark_op);
2224 }
2225 // The collector state may be any legal state at this point
2226 // since the background collector may have yielded to the
2227 // foreground collector.
2228 break;
2229 case Marking:
2230 // initial marking in checkpointRootsInitialWork has been completed
2231 if (markFromRoots(true)) { // we were successful
2232 assert(_collectorState == Precleaning, "Collector state should "
2233 "have changed");
2234 } else {
2235 assert(_foregroundGCIsActive, "Internal state inconsistency");
2236 }
2237 break;
2238 case Precleaning:
2239 if (UseAdaptiveSizePolicy) {
2240 size_policy()->concurrent_precleaning_begin();
2241 }
2242 // marking from roots in markFromRoots has been completed
2243 preclean();
2244 if (UseAdaptiveSizePolicy) {
2245 size_policy()->concurrent_precleaning_end();
2246 }
2247 assert(_collectorState == AbortablePreclean ||
2248 _collectorState == FinalMarking,
2249 "Collector state should have changed");
2250 break;
2251 case AbortablePreclean:
2252 if (UseAdaptiveSizePolicy) {
2253 size_policy()->concurrent_phases_resume();
2254 }
2255 abortable_preclean();
2256 if (UseAdaptiveSizePolicy) {
2257 size_policy()->concurrent_precleaning_end();
2258 }
2259 assert(_collectorState == FinalMarking, "Collector state should "
2260 "have changed");
2261 break;
2262 case FinalMarking:
2263 {
2264 ReleaseForegroundGC x(this);
2266 VM_CMS_Final_Remark final_remark_op(this);
2267 VMThread::execute(&final_remark_op);
2268 }
2269 assert(_foregroundGCShouldWait, "block post-condition");
2270 break;
2271 case Sweeping:
2272 if (UseAdaptiveSizePolicy) {
2273 size_policy()->concurrent_sweeping_begin();
2274 }
2275 // final marking in checkpointRootsFinal has been completed
2276 sweep(true);
2277 assert(_collectorState == Resizing, "Collector state change "
2278 "to Resizing must be done under the free_list_lock");
2279 _full_gcs_since_conc_gc = 0;
2281 // Stop the timers for adaptive size policy for the concurrent phases
2282 if (UseAdaptiveSizePolicy) {
2283 size_policy()->concurrent_sweeping_end();
2284 size_policy()->concurrent_phases_end(gch->gc_cause(),
2285 gch->prev_gen(_cmsGen)->capacity(),
2286 _cmsGen->free());
2287 }
2289 case Resizing: {
2290 // Sweeping has been completed...
2291 // At this point the background collection has completed.
2292 // Don't move the call to compute_new_size() down
2293 // into code that might be executed if the background
2294 // collection was preempted.
2295 {
2296 ReleaseForegroundGC x(this); // unblock FG collection
2297 MutexLockerEx y(Heap_lock, Mutex::_no_safepoint_check_flag);
2298 CMSTokenSync z(true); // not strictly needed.
2299 if (_collectorState == Resizing) {
2300 compute_new_size();
2301 _collectorState = Resetting;
2302 } else {
2303 assert(_collectorState == Idling, "The state should only change"
2304 " because the foreground collector has finished the collection");
2305 }
2306 }
2307 break;
2308 }
2309 case Resetting:
2310 // CMS heap resizing has been completed
2311 reset(true);
2312 assert(_collectorState == Idling, "Collector state should "
2313 "have changed");
2314 stats().record_cms_end();
2315 // Don't move the concurrent_phases_end() and compute_new_size()
2316 // calls to here because a preempted background collection
2317 // has it's state set to "Resetting".
2318 break;
2319 case Idling:
2320 default:
2321 ShouldNotReachHere();
2322 break;
2323 }
2324 if (TraceCMSState) {
2325 gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
2326 Thread::current(), _collectorState);
2327 }
2328 assert(_foregroundGCShouldWait, "block post-condition");
2329 }
2331 // Should this be in gc_epilogue?
2332 collector_policy()->counters()->update_counters();
2334 {
2335 // Clear _foregroundGCShouldWait and, in the event that the
2336 // foreground collector is waiting, notify it, before
2337 // returning.
2338 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2339 _foregroundGCShouldWait = false;
2340 if (_foregroundGCIsActive) {
2341 CGC_lock->notify();
2342 }
2343 assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
2344 "Possible deadlock");
2345 }
2346 if (TraceCMSState) {
2347 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
2348 " exiting collection CMS state %d",
2349 Thread::current(), _collectorState);
2350 }
2351 if (PrintGC && Verbose) {
2352 _cmsGen->print_heap_change(prev_used);
2353 }
2354 }
2356 void CMSCollector::collect_in_foreground(bool clear_all_soft_refs) {
2357 assert(_foregroundGCIsActive && !_foregroundGCShouldWait,
2358 "Foreground collector should be waiting, not executing");
2359 assert(Thread::current()->is_VM_thread(), "A foreground collection"
2360 "may only be done by the VM Thread with the world stopped");
2361 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
2362 "VM thread should have CMS token");
2364 NOT_PRODUCT(TraceTime t("CMS:MS (foreground) ", PrintGCDetails && Verbose,
2365 true, gclog_or_tty);)
2366 if (UseAdaptiveSizePolicy) {
2367 size_policy()->ms_collection_begin();
2368 }
2369 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact);
2371 HandleMark hm; // Discard invalid handles created during verification
2373 if (VerifyBeforeGC &&
2374 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2375 Universe::verify(true);
2376 }
2378 bool init_mark_was_synchronous = false; // until proven otherwise
2379 while (_collectorState != Idling) {
2380 if (TraceCMSState) {
2381 gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
2382 Thread::current(), _collectorState);
2383 }
2384 switch (_collectorState) {
2385 case InitialMarking:
2386 init_mark_was_synchronous = true; // fact to be exploited in re-mark
2387 checkpointRootsInitial(false);
2388 assert(_collectorState == Marking, "Collector state should have changed"
2389 " within checkpointRootsInitial()");
2390 break;
2391 case Marking:
2392 // initial marking in checkpointRootsInitialWork has been completed
2393 if (VerifyDuringGC &&
2394 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2395 gclog_or_tty->print("Verify before initial mark: ");
2396 Universe::verify(true);
2397 }
2398 {
2399 bool res = markFromRoots(false);
2400 assert(res && _collectorState == FinalMarking, "Collector state should "
2401 "have changed");
2402 break;
2403 }
2404 case FinalMarking:
2405 if (VerifyDuringGC &&
2406 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2407 gclog_or_tty->print("Verify before re-mark: ");
2408 Universe::verify(true);
2409 }
2410 checkpointRootsFinal(false, clear_all_soft_refs,
2411 init_mark_was_synchronous);
2412 assert(_collectorState == Sweeping, "Collector state should not "
2413 "have changed within checkpointRootsFinal()");
2414 break;
2415 case Sweeping:
2416 // final marking in checkpointRootsFinal has been completed
2417 if (VerifyDuringGC &&
2418 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2419 gclog_or_tty->print("Verify before sweep: ");
2420 Universe::verify(true);
2421 }
2422 sweep(false);
2423 assert(_collectorState == Resizing, "Incorrect state");
2424 break;
2425 case Resizing: {
2426 // Sweeping has been completed; the actual resize in this case
2427 // is done separately; nothing to be done in this state.
2428 _collectorState = Resetting;
2429 break;
2430 }
2431 case Resetting:
2432 // The heap has been resized.
2433 if (VerifyDuringGC &&
2434 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2435 gclog_or_tty->print("Verify before reset: ");
2436 Universe::verify(true);
2437 }
2438 reset(false);
2439 assert(_collectorState == Idling, "Collector state should "
2440 "have changed");
2441 break;
2442 case Precleaning:
2443 case AbortablePreclean:
2444 // Elide the preclean phase
2445 _collectorState = FinalMarking;
2446 break;
2447 default:
2448 ShouldNotReachHere();
2449 }
2450 if (TraceCMSState) {
2451 gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
2452 Thread::current(), _collectorState);
2453 }
2454 }
2456 if (UseAdaptiveSizePolicy) {
2457 GenCollectedHeap* gch = GenCollectedHeap::heap();
2458 size_policy()->ms_collection_end(gch->gc_cause());
2459 }
2461 if (VerifyAfterGC &&
2462 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2463 Universe::verify(true);
2464 }
2465 if (TraceCMSState) {
2466 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
2467 " exiting collection CMS state %d",
2468 Thread::current(), _collectorState);
2469 }
2470 }
2472 bool CMSCollector::waitForForegroundGC() {
2473 bool res = false;
2474 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
2475 "CMS thread should have CMS token");
2476 // Block the foreground collector until the
2477 // background collectors decides whether to
2478 // yield.
2479 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2480 _foregroundGCShouldWait = true;
2481 if (_foregroundGCIsActive) {
2482 // The background collector yields to the
2483 // foreground collector and returns a value
2484 // indicating that it has yielded. The foreground
2485 // collector can proceed.
2486 res = true;
2487 _foregroundGCShouldWait = false;
2488 ConcurrentMarkSweepThread::clear_CMS_flag(
2489 ConcurrentMarkSweepThread::CMS_cms_has_token);
2490 ConcurrentMarkSweepThread::set_CMS_flag(
2491 ConcurrentMarkSweepThread::CMS_cms_wants_token);
2492 // Get a possibly blocked foreground thread going
2493 CGC_lock->notify();
2494 if (TraceCMSState) {
2495 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d",
2496 Thread::current(), _collectorState);
2497 }
2498 while (_foregroundGCIsActive) {
2499 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
2500 }
2501 ConcurrentMarkSweepThread::set_CMS_flag(
2502 ConcurrentMarkSweepThread::CMS_cms_has_token);
2503 ConcurrentMarkSweepThread::clear_CMS_flag(
2504 ConcurrentMarkSweepThread::CMS_cms_wants_token);
2505 }
2506 if (TraceCMSState) {
2507 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d",
2508 Thread::current(), _collectorState);
2509 }
2510 return res;
2511 }
2513 // Because of the need to lock the free lists and other structures in
2514 // the collector, common to all the generations that the collector is
2515 // collecting, we need the gc_prologues of individual CMS generations
2516 // delegate to their collector. It may have been simpler had the
2517 // current infrastructure allowed one to call a prologue on a
2518 // collector. In the absence of that we have the generation's
2519 // prologue delegate to the collector, which delegates back
2520 // some "local" work to a worker method in the individual generations
2521 // that it's responsible for collecting, while itself doing any
2522 // work common to all generations it's responsible for. A similar
2523 // comment applies to the gc_epilogue()'s.
2524 // The role of the varaible _between_prologue_and_epilogue is to
2525 // enforce the invocation protocol.
2526 void CMSCollector::gc_prologue(bool full) {
2527 // Call gc_prologue_work() for each CMSGen and PermGen that
2528 // we are responsible for.
2530 // The following locking discipline assumes that we are only called
2531 // when the world is stopped.
2532 assert(SafepointSynchronize::is_at_safepoint(), "world is stopped assumption");
2534 // The CMSCollector prologue must call the gc_prologues for the
2535 // "generations" (including PermGen if any) that it's responsible
2536 // for.
2538 assert( Thread::current()->is_VM_thread()
2539 || ( CMSScavengeBeforeRemark
2540 && Thread::current()->is_ConcurrentGC_thread()),
2541 "Incorrect thread type for prologue execution");
2543 if (_between_prologue_and_epilogue) {
2544 // We have already been invoked; this is a gc_prologue delegation
2545 // from yet another CMS generation that we are responsible for, just
2546 // ignore it since all relevant work has already been done.
2547 return;
2548 }
2550 // set a bit saying prologue has been called; cleared in epilogue
2551 _between_prologue_and_epilogue = true;
2552 // Claim locks for common data structures, then call gc_prologue_work()
2553 // for each CMSGen and PermGen that we are responsible for.
2555 getFreelistLocks(); // gets free list locks on constituent spaces
2556 bitMapLock()->lock_without_safepoint_check();
2558 // Should call gc_prologue_work() for all cms gens we are responsible for
2559 bool registerClosure = _collectorState >= Marking
2560 && _collectorState < Sweeping;
2561 ModUnionClosure* muc = ParallelGCThreads > 0 ? &_modUnionClosurePar
2562 : &_modUnionClosure;
2563 _cmsGen->gc_prologue_work(full, registerClosure, muc);
2564 _permGen->gc_prologue_work(full, registerClosure, muc);
2566 if (!full) {
2567 stats().record_gc0_begin();
2568 }
2569 }
2571 void ConcurrentMarkSweepGeneration::gc_prologue(bool full) {
2572 // Delegate to CMScollector which knows how to coordinate between
2573 // this and any other CMS generations that it is responsible for
2574 // collecting.
2575 collector()->gc_prologue(full);
2576 }
2578 // This is a "private" interface for use by this generation's CMSCollector.
2579 // Not to be called directly by any other entity (for instance,
2580 // GenCollectedHeap, which calls the "public" gc_prologue method above).
2581 void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
2582 bool registerClosure, ModUnionClosure* modUnionClosure) {
2583 assert(!incremental_collection_failed(), "Shouldn't be set yet");
2584 assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,
2585 "Should be NULL");
2586 if (registerClosure) {
2587 cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure);
2588 }
2589 cmsSpace()->gc_prologue();
2590 // Clear stat counters
2591 NOT_PRODUCT(
2592 assert(_numObjectsPromoted == 0, "check");
2593 assert(_numWordsPromoted == 0, "check");
2594 if (Verbose && PrintGC) {
2595 gclog_or_tty->print("Allocated "SIZE_FORMAT" objects, "
2596 SIZE_FORMAT" bytes concurrently",
2597 _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord));
2598 }
2599 _numObjectsAllocated = 0;
2600 _numWordsAllocated = 0;
2601 )
2602 }
2604 void CMSCollector::gc_epilogue(bool full) {
2605 // The following locking discipline assumes that we are only called
2606 // when the world is stopped.
2607 assert(SafepointSynchronize::is_at_safepoint(),
2608 "world is stopped assumption");
2610 // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks
2611 // if linear allocation blocks need to be appropriately marked to allow the
2612 // the blocks to be parsable. We also check here whether we need to nudge the
2613 // CMS collector thread to start a new cycle (if it's not already active).
2614 assert( Thread::current()->is_VM_thread()
2615 || ( CMSScavengeBeforeRemark
2616 && Thread::current()->is_ConcurrentGC_thread()),
2617 "Incorrect thread type for epilogue execution");
2619 if (!_between_prologue_and_epilogue) {
2620 // We have already been invoked; this is a gc_epilogue delegation
2621 // from yet another CMS generation that we are responsible for, just
2622 // ignore it since all relevant work has already been done.
2623 return;
2624 }
2625 assert(haveFreelistLocks(), "must have freelist locks");
2626 assert_lock_strong(bitMapLock());
2628 _cmsGen->gc_epilogue_work(full);
2629 _permGen->gc_epilogue_work(full);
2631 if (_collectorState == AbortablePreclean || _collectorState == Precleaning) {
2632 // in case sampling was not already enabled, enable it
2633 _start_sampling = true;
2634 }
2635 // reset _eden_chunk_array so sampling starts afresh
2636 _eden_chunk_index = 0;
2638 size_t cms_used = _cmsGen->cmsSpace()->used();
2639 size_t perm_used = _permGen->cmsSpace()->used();
2641 // update performance counters - this uses a special version of
2642 // update_counters() that allows the utilization to be passed as a
2643 // parameter, avoiding multiple calls to used().
2644 //
2645 _cmsGen->update_counters(cms_used);
2646 _permGen->update_counters(perm_used);
2648 if (CMSIncrementalMode) {
2649 icms_update_allocation_limits();
2650 }
2652 bitMapLock()->unlock();
2653 releaseFreelistLocks();
2655 _between_prologue_and_epilogue = false; // ready for next cycle
2656 }
2658 void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
2659 collector()->gc_epilogue(full);
2661 // Also reset promotion tracking in par gc thread states.
2662 if (ParallelGCThreads > 0) {
2663 for (uint i = 0; i < ParallelGCThreads; i++) {
2664 _par_gc_thread_states[i]->promo.stopTrackingPromotions();
2665 }
2666 }
2667 }
2669 void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) {
2670 assert(!incremental_collection_failed(), "Should have been cleared");
2671 cmsSpace()->setPreconsumptionDirtyCardClosure(NULL);
2672 cmsSpace()->gc_epilogue();
2673 // Print stat counters
2674 NOT_PRODUCT(
2675 assert(_numObjectsAllocated == 0, "check");
2676 assert(_numWordsAllocated == 0, "check");
2677 if (Verbose && PrintGC) {
2678 gclog_or_tty->print("Promoted "SIZE_FORMAT" objects, "
2679 SIZE_FORMAT" bytes",
2680 _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord));
2681 }
2682 _numObjectsPromoted = 0;
2683 _numWordsPromoted = 0;
2684 )
2686 if (PrintGC && Verbose) {
2687 // Call down the chain in contiguous_available needs the freelistLock
2688 // so print this out before releasing the freeListLock.
2689 gclog_or_tty->print(" Contiguous available "SIZE_FORMAT" bytes ",
2690 contiguous_available());
2691 }
2692 }
2694 #ifndef PRODUCT
2695 bool CMSCollector::have_cms_token() {
2696 Thread* thr = Thread::current();
2697 if (thr->is_VM_thread()) {
2698 return ConcurrentMarkSweepThread::vm_thread_has_cms_token();
2699 } else if (thr->is_ConcurrentGC_thread()) {
2700 return ConcurrentMarkSweepThread::cms_thread_has_cms_token();
2701 } else if (thr->is_GC_task_thread()) {
2702 return ConcurrentMarkSweepThread::vm_thread_has_cms_token() &&
2703 ParGCRareEvent_lock->owned_by_self();
2704 }
2705 return false;
2706 }
2707 #endif
2709 // Check reachability of the given heap address in CMS generation,
2710 // treating all other generations as roots.
2711 bool CMSCollector::is_cms_reachable(HeapWord* addr) {
2712 // We could "guarantee" below, rather than assert, but i'll
2713 // leave these as "asserts" so that an adventurous debugger
2714 // could try this in the product build provided some subset of
2715 // the conditions were met, provided they were intersted in the
2716 // results and knew that the computation below wouldn't interfere
2717 // with other concurrent computations mutating the structures
2718 // being read or written.
2719 assert(SafepointSynchronize::is_at_safepoint(),
2720 "Else mutations in object graph will make answer suspect");
2721 assert(have_cms_token(), "Should hold cms token");
2722 assert(haveFreelistLocks(), "must hold free list locks");
2723 assert_lock_strong(bitMapLock());
2725 // Clear the marking bit map array before starting, but, just
2726 // for kicks, first report if the given address is already marked
2727 gclog_or_tty->print_cr("Start: Address 0x%x is%s marked", addr,
2728 _markBitMap.isMarked(addr) ? "" : " not");
2730 if (verify_after_remark()) {
2731 MutexLockerEx x(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2732 bool result = verification_mark_bm()->isMarked(addr);
2733 gclog_or_tty->print_cr("TransitiveMark: Address 0x%x %s marked", addr,
2734 result ? "IS" : "is NOT");
2735 return result;
2736 } else {
2737 gclog_or_tty->print_cr("Could not compute result");
2738 return false;
2739 }
2740 }
2742 ////////////////////////////////////////////////////////
2743 // CMS Verification Support
2744 ////////////////////////////////////////////////////////
2745 // Following the remark phase, the following invariant
2746 // should hold -- each object in the CMS heap which is
2747 // marked in markBitMap() should be marked in the verification_mark_bm().
2749 class VerifyMarkedClosure: public BitMapClosure {
2750 CMSBitMap* _marks;
2751 bool _failed;
2753 public:
2754 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2756 void do_bit(size_t offset) {
2757 HeapWord* addr = _marks->offsetToHeapWord(offset);
2758 if (!_marks->isMarked(addr)) {
2759 oop(addr)->print();
2760 gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", addr);
2761 _failed = true;
2762 }
2763 }
2765 bool failed() { return _failed; }
2766 };
2768 bool CMSCollector::verify_after_remark() {
2769 gclog_or_tty->print(" [Verifying CMS Marking... ");
2770 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2771 static bool init = false;
2773 assert(SafepointSynchronize::is_at_safepoint(),
2774 "Else mutations in object graph will make answer suspect");
2775 assert(have_cms_token(),
2776 "Else there may be mutual interference in use of "
2777 " verification data structures");
2778 assert(_collectorState > Marking && _collectorState <= Sweeping,
2779 "Else marking info checked here may be obsolete");
2780 assert(haveFreelistLocks(), "must hold free list locks");
2781 assert_lock_strong(bitMapLock());
2784 // Allocate marking bit map if not already allocated
2785 if (!init) { // first time
2786 if (!verification_mark_bm()->allocate(_span)) {
2787 return false;
2788 }
2789 init = true;
2790 }
2792 assert(verification_mark_stack()->isEmpty(), "Should be empty");
2794 // Turn off refs discovery -- so we will be tracing through refs.
2795 // This is as intended, because by this time
2796 // GC must already have cleared any refs that need to be cleared,
2797 // and traced those that need to be marked; moreover,
2798 // the marking done here is not going to intefere in any
2799 // way with the marking information used by GC.
2800 NoRefDiscovery no_discovery(ref_processor());
2802 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
2804 // Clear any marks from a previous round
2805 verification_mark_bm()->clear_all();
2806 assert(verification_mark_stack()->isEmpty(), "markStack should be empty");
2807 assert(overflow_list_is_empty(), "overflow list should be empty");
2809 GenCollectedHeap* gch = GenCollectedHeap::heap();
2810 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2811 // Update the saved marks which may affect the root scans.
2812 gch->save_marks();
2814 if (CMSRemarkVerifyVariant == 1) {
2815 // In this first variant of verification, we complete
2816 // all marking, then check if the new marks-verctor is
2817 // a subset of the CMS marks-vector.
2818 verify_after_remark_work_1();
2819 } else if (CMSRemarkVerifyVariant == 2) {
2820 // In this second variant of verification, we flag an error
2821 // (i.e. an object reachable in the new marks-vector not reachable
2822 // in the CMS marks-vector) immediately, also indicating the
2823 // identify of an object (A) that references the unmarked object (B) --
2824 // presumably, a mutation to A failed to be picked up by preclean/remark?
2825 verify_after_remark_work_2();
2826 } else {
2827 warning("Unrecognized value %d for CMSRemarkVerifyVariant",
2828 CMSRemarkVerifyVariant);
2829 }
2830 gclog_or_tty->print(" done] ");
2831 return true;
2832 }
2834 void CMSCollector::verify_after_remark_work_1() {
2835 ResourceMark rm;
2836 HandleMark hm;
2837 GenCollectedHeap* gch = GenCollectedHeap::heap();
2839 // Mark from roots one level into CMS
2840 MarkRefsIntoClosure notOlder(_span, verification_mark_bm(), true /* nmethods */);
2841 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2843 gch->gen_process_strong_roots(_cmsGen->level(),
2844 true, // younger gens are roots
2845 true, // collecting perm gen
2846 SharedHeap::ScanningOption(roots_scanning_options()),
2847 NULL, ¬Older);
2849 // Now mark from the roots
2850 assert(_revisitStack.isEmpty(), "Should be empty");
2851 MarkFromRootsClosure markFromRootsClosure(this, _span,
2852 verification_mark_bm(), verification_mark_stack(), &_revisitStack,
2853 false /* don't yield */, true /* verifying */);
2854 assert(_restart_addr == NULL, "Expected pre-condition");
2855 verification_mark_bm()->iterate(&markFromRootsClosure);
2856 while (_restart_addr != NULL) {
2857 // Deal with stack overflow: by restarting at the indicated
2858 // address.
2859 HeapWord* ra = _restart_addr;
2860 markFromRootsClosure.reset(ra);
2861 _restart_addr = NULL;
2862 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2863 }
2864 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2865 verify_work_stacks_empty();
2866 // Should reset the revisit stack above, since no class tree
2867 // surgery is forthcoming.
2868 _revisitStack.reset(); // throwing away all contents
2870 // Marking completed -- now verify that each bit marked in
2871 // verification_mark_bm() is also marked in markBitMap(); flag all
2872 // errors by printing corresponding objects.
2873 VerifyMarkedClosure vcl(markBitMap());
2874 verification_mark_bm()->iterate(&vcl);
2875 if (vcl.failed()) {
2876 gclog_or_tty->print("Verification failed");
2877 Universe::heap()->print();
2878 fatal(" ... aborting");
2879 }
2880 }
2882 void CMSCollector::verify_after_remark_work_2() {
2883 ResourceMark rm;
2884 HandleMark hm;
2885 GenCollectedHeap* gch = GenCollectedHeap::heap();
2887 // Mark from roots one level into CMS
2888 MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(),
2889 markBitMap(), true /* nmethods */);
2890 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2891 gch->gen_process_strong_roots(_cmsGen->level(),
2892 true, // younger gens are roots
2893 true, // collecting perm gen
2894 SharedHeap::ScanningOption(roots_scanning_options()),
2895 NULL, ¬Older);
2897 // Now mark from the roots
2898 assert(_revisitStack.isEmpty(), "Should be empty");
2899 MarkFromRootsVerifyClosure markFromRootsClosure(this, _span,
2900 verification_mark_bm(), markBitMap(), verification_mark_stack());
2901 assert(_restart_addr == NULL, "Expected pre-condition");
2902 verification_mark_bm()->iterate(&markFromRootsClosure);
2903 while (_restart_addr != NULL) {
2904 // Deal with stack overflow: by restarting at the indicated
2905 // address.
2906 HeapWord* ra = _restart_addr;
2907 markFromRootsClosure.reset(ra);
2908 _restart_addr = NULL;
2909 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2910 }
2911 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2912 verify_work_stacks_empty();
2913 // Should reset the revisit stack above, since no class tree
2914 // surgery is forthcoming.
2915 _revisitStack.reset(); // throwing away all contents
2917 // Marking completed -- now verify that each bit marked in
2918 // verification_mark_bm() is also marked in markBitMap(); flag all
2919 // errors by printing corresponding objects.
2920 VerifyMarkedClosure vcl(markBitMap());
2921 verification_mark_bm()->iterate(&vcl);
2922 assert(!vcl.failed(), "Else verification above should not have succeeded");
2923 }
2925 void ConcurrentMarkSweepGeneration::save_marks() {
2926 // delegate to CMS space
2927 cmsSpace()->save_marks();
2928 for (uint i = 0; i < ParallelGCThreads; i++) {
2929 _par_gc_thread_states[i]->promo.startTrackingPromotions();
2930 }
2931 }
2933 bool ConcurrentMarkSweepGeneration::no_allocs_since_save_marks() {
2934 return cmsSpace()->no_allocs_since_save_marks();
2935 }
2937 #define CMS_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \
2938 \
2939 void ConcurrentMarkSweepGeneration:: \
2940 oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \
2941 cl->set_generation(this); \
2942 cmsSpace()->oop_since_save_marks_iterate##nv_suffix(cl); \
2943 cl->reset_generation(); \
2944 save_marks(); \
2945 }
2947 ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DEFN)
2949 void
2950 ConcurrentMarkSweepGeneration::object_iterate_since_last_GC(ObjectClosure* blk)
2951 {
2952 // Not currently implemented; need to do the following. -- ysr.
2953 // dld -- I think that is used for some sort of allocation profiler. So it
2954 // really means the objects allocated by the mutator since the last
2955 // GC. We could potentially implement this cheaply by recording only
2956 // the direct allocations in a side data structure.
2957 //
2958 // I think we probably ought not to be required to support these
2959 // iterations at any arbitrary point; I think there ought to be some
2960 // call to enable/disable allocation profiling in a generation/space,
2961 // and the iterator ought to return the objects allocated in the
2962 // gen/space since the enable call, or the last iterator call (which
2963 // will probably be at a GC.) That way, for gens like CM&S that would
2964 // require some extra data structure to support this, we only pay the
2965 // cost when it's in use...
2966 cmsSpace()->object_iterate_since_last_GC(blk);
2967 }
2969 void
2970 ConcurrentMarkSweepGeneration::younger_refs_iterate(OopsInGenClosure* cl) {
2971 cl->set_generation(this);
2972 younger_refs_in_space_iterate(_cmsSpace, cl);
2973 cl->reset_generation();
2974 }
2976 void
2977 ConcurrentMarkSweepGeneration::oop_iterate(MemRegion mr, OopClosure* cl) {
2978 if (freelistLock()->owned_by_self()) {
2979 Generation::oop_iterate(mr, cl);
2980 } else {
2981 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
2982 Generation::oop_iterate(mr, cl);
2983 }
2984 }
2986 void
2987 ConcurrentMarkSweepGeneration::oop_iterate(OopClosure* cl) {
2988 if (freelistLock()->owned_by_self()) {
2989 Generation::oop_iterate(cl);
2990 } else {
2991 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
2992 Generation::oop_iterate(cl);
2993 }
2994 }
2996 void
2997 ConcurrentMarkSweepGeneration::object_iterate(ObjectClosure* cl) {
2998 if (freelistLock()->owned_by_self()) {
2999 Generation::object_iterate(cl);
3000 } else {
3001 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
3002 Generation::object_iterate(cl);
3003 }
3004 }
3006 void
3007 ConcurrentMarkSweepGeneration::pre_adjust_pointers() {
3008 }
3010 void
3011 ConcurrentMarkSweepGeneration::post_compact() {
3012 }
3014 void
3015 ConcurrentMarkSweepGeneration::prepare_for_verify() {
3016 // Fix the linear allocation blocks to look like free blocks.
3018 // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those
3019 // are not called when the heap is verified during universe initialization and
3020 // at vm shutdown.
3021 if (freelistLock()->owned_by_self()) {
3022 cmsSpace()->prepare_for_verify();
3023 } else {
3024 MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag);
3025 cmsSpace()->prepare_for_verify();
3026 }
3027 }
3029 void
3030 ConcurrentMarkSweepGeneration::verify(bool allow_dirty /* ignored */) {
3031 // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those
3032 // are not called when the heap is verified during universe initialization and
3033 // at vm shutdown.
3034 if (freelistLock()->owned_by_self()) {
3035 cmsSpace()->verify(false /* ignored */);
3036 } else {
3037 MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag);
3038 cmsSpace()->verify(false /* ignored */);
3039 }
3040 }
3042 void CMSCollector::verify(bool allow_dirty /* ignored */) {
3043 _cmsGen->verify(allow_dirty);
3044 _permGen->verify(allow_dirty);
3045 }
3047 #ifndef PRODUCT
3048 bool CMSCollector::overflow_list_is_empty() const {
3049 assert(_num_par_pushes >= 0, "Inconsistency");
3050 if (_overflow_list == NULL) {
3051 assert(_num_par_pushes == 0, "Inconsistency");
3052 }
3053 return _overflow_list == NULL;
3054 }
3056 // The methods verify_work_stacks_empty() and verify_overflow_empty()
3057 // merely consolidate assertion checks that appear to occur together frequently.
3058 void CMSCollector::verify_work_stacks_empty() const {
3059 assert(_markStack.isEmpty(), "Marking stack should be empty");
3060 assert(overflow_list_is_empty(), "Overflow list should be empty");
3061 }
3063 void CMSCollector::verify_overflow_empty() const {
3064 assert(overflow_list_is_empty(), "Overflow list should be empty");
3065 assert(no_preserved_marks(), "No preserved marks");
3066 }
3067 #endif // PRODUCT
3069 // Decide if we want to enable class unloading as part of the
3070 // ensuing concurrent GC cycle. We will collect the perm gen and
3071 // unload classes if it's the case that:
3072 // (1) an explicit gc request has been made and the flag
3073 // ExplicitGCInvokesConcurrentAndUnloadsClasses is set, OR
3074 // (2) (a) class unloading is enabled at the command line, and
3075 // (b) (i) perm gen threshold has been crossed, or
3076 // (ii) old gen is getting really full, or
3077 // (iii) the previous N CMS collections did not collect the
3078 // perm gen
3079 // NOTE: Provided there is no change in the state of the heap between
3080 // calls to this method, it should have idempotent results. Moreover,
3081 // its results should be monotonically increasing (i.e. going from 0 to 1,
3082 // but not 1 to 0) between successive calls between which the heap was
3083 // not collected. For the implementation below, it must thus rely on
3084 // the property that concurrent_cycles_since_last_unload()
3085 // will not decrease unless a collection cycle happened and that
3086 // _permGen->should_concurrent_collect() and _cmsGen->is_too_full() are
3087 // themselves also monotonic in that sense. See check_monotonicity()
3088 // below.
3089 bool CMSCollector::update_should_unload_classes() {
3090 _should_unload_classes = false;
3091 // Condition 1 above
3092 if (_full_gc_requested && ExplicitGCInvokesConcurrentAndUnloadsClasses) {
3093 _should_unload_classes = true;
3094 } else if (CMSClassUnloadingEnabled) { // Condition 2.a above
3095 // Disjuncts 2.b.(i,ii,iii) above
3096 _should_unload_classes = (concurrent_cycles_since_last_unload() >=
3097 CMSClassUnloadingMaxInterval)
3098 || _permGen->should_concurrent_collect()
3099 || _cmsGen->is_too_full();
3100 }
3101 return _should_unload_classes;
3102 }
3104 bool ConcurrentMarkSweepGeneration::is_too_full() const {
3105 bool res = should_concurrent_collect();
3106 res = res && (occupancy() > (double)CMSIsTooFullPercentage/100.0);
3107 return res;
3108 }
3110 void CMSCollector::setup_cms_unloading_and_verification_state() {
3111 const bool should_verify = VerifyBeforeGC || VerifyAfterGC || VerifyDuringGC
3112 || VerifyBeforeExit;
3113 const int rso = SharedHeap::SO_Symbols | SharedHeap::SO_Strings
3114 | SharedHeap::SO_CodeCache;
3116 if (should_unload_classes()) { // Should unload classes this cycle
3117 remove_root_scanning_option(rso); // Shrink the root set appropriately
3118 set_verifying(should_verify); // Set verification state for this cycle
3119 return; // Nothing else needs to be done at this time
3120 }
3122 // Not unloading classes this cycle
3123 assert(!should_unload_classes(), "Inconsitency!");
3124 if ((!verifying() || unloaded_classes_last_cycle()) && should_verify) {
3125 // We were not verifying, or we _were_ unloading classes in the last cycle,
3126 // AND some verification options are enabled this cycle; in this case,
3127 // we must make sure that the deadness map is allocated if not already so,
3128 // and cleared (if already allocated previously --
3129 // CMSBitMap::sizeInBits() is used to determine if it's allocated).
3130 if (perm_gen_verify_bit_map()->sizeInBits() == 0) {
3131 if (!perm_gen_verify_bit_map()->allocate(_permGen->reserved())) {
3132 warning("Failed to allocate permanent generation verification CMS Bit Map;\n"
3133 "permanent generation verification disabled");
3134 return; // Note that we leave verification disabled, so we'll retry this
3135 // allocation next cycle. We _could_ remember this failure
3136 // and skip further attempts and permanently disable verification
3137 // attempts if that is considered more desirable.
3138 }
3139 assert(perm_gen_verify_bit_map()->covers(_permGen->reserved()),
3140 "_perm_gen_ver_bit_map inconsistency?");
3141 } else {
3142 perm_gen_verify_bit_map()->clear_all();
3143 }
3144 // Include symbols, strings and code cache elements to prevent their resurrection.
3145 add_root_scanning_option(rso);
3146 set_verifying(true);
3147 } else if (verifying() && !should_verify) {
3148 // We were verifying, but some verification flags got disabled.
3149 set_verifying(false);
3150 // Exclude symbols, strings and code cache elements from root scanning to
3151 // reduce IM and RM pauses.
3152 remove_root_scanning_option(rso);
3153 }
3154 }
3157 #ifndef PRODUCT
3158 HeapWord* CMSCollector::block_start(const void* p) const {
3159 const HeapWord* addr = (HeapWord*)p;
3160 if (_span.contains(p)) {
3161 if (_cmsGen->cmsSpace()->is_in_reserved(addr)) {
3162 return _cmsGen->cmsSpace()->block_start(p);
3163 } else {
3164 assert(_permGen->cmsSpace()->is_in_reserved(addr),
3165 "Inconsistent _span?");
3166 return _permGen->cmsSpace()->block_start(p);
3167 }
3168 }
3169 return NULL;
3170 }
3171 #endif
3173 HeapWord*
3174 ConcurrentMarkSweepGeneration::expand_and_allocate(size_t word_size,
3175 bool tlab,
3176 bool parallel) {
3177 assert(!tlab, "Can't deal with TLAB allocation");
3178 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
3179 expand(word_size*HeapWordSize, MinHeapDeltaBytes,
3180 CMSExpansionCause::_satisfy_allocation);
3181 if (GCExpandToAllocateDelayMillis > 0) {
3182 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
3183 }
3184 return have_lock_and_allocate(word_size, tlab);
3185 }
3187 // YSR: All of this generation expansion/shrinking stuff is an exact copy of
3188 // OneContigSpaceCardGeneration, which makes me wonder if we should move this
3189 // to CardGeneration and share it...
3190 void ConcurrentMarkSweepGeneration::expand(size_t bytes, size_t expand_bytes,
3191 CMSExpansionCause::Cause cause)
3192 {
3193 assert_locked_or_safepoint(Heap_lock);
3195 size_t aligned_bytes = ReservedSpace::page_align_size_up(bytes);
3196 size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes);
3197 bool success = false;
3198 if (aligned_expand_bytes > aligned_bytes) {
3199 success = grow_by(aligned_expand_bytes);
3200 }
3201 if (!success) {
3202 success = grow_by(aligned_bytes);
3203 }
3204 if (!success) {
3205 size_t remaining_bytes = _virtual_space.uncommitted_size();
3206 if (remaining_bytes > 0) {
3207 success = grow_by(remaining_bytes);
3208 }
3209 }
3210 if (GC_locker::is_active()) {
3211 if (PrintGC && Verbose) {
3212 gclog_or_tty->print_cr("Garbage collection disabled, expanded heap instead");
3213 }
3214 }
3215 // remember why we expanded; this information is used
3216 // by shouldConcurrentCollect() when making decisions on whether to start
3217 // a new CMS cycle.
3218 if (success) {
3219 set_expansion_cause(cause);
3220 if (PrintGCDetails && Verbose) {
3221 gclog_or_tty->print_cr("Expanded CMS gen for %s",
3222 CMSExpansionCause::to_string(cause));
3223 }
3224 }
3225 }
3227 HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) {
3228 HeapWord* res = NULL;
3229 MutexLocker x(ParGCRareEvent_lock);
3230 while (true) {
3231 // Expansion by some other thread might make alloc OK now:
3232 res = ps->lab.alloc(word_sz);
3233 if (res != NULL) return res;
3234 // If there's not enough expansion space available, give up.
3235 if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) {
3236 return NULL;
3237 }
3238 // Otherwise, we try expansion.
3239 expand(word_sz*HeapWordSize, MinHeapDeltaBytes,
3240 CMSExpansionCause::_allocate_par_lab);
3241 // Now go around the loop and try alloc again;
3242 // A competing par_promote might beat us to the expansion space,
3243 // so we may go around the loop again if promotion fails agaion.
3244 if (GCExpandToAllocateDelayMillis > 0) {
3245 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
3246 }
3247 }
3248 }
3251 bool ConcurrentMarkSweepGeneration::expand_and_ensure_spooling_space(
3252 PromotionInfo* promo) {
3253 MutexLocker x(ParGCRareEvent_lock);
3254 size_t refill_size_bytes = promo->refillSize() * HeapWordSize;
3255 while (true) {
3256 // Expansion by some other thread might make alloc OK now:
3257 if (promo->ensure_spooling_space()) {
3258 assert(promo->has_spooling_space(),
3259 "Post-condition of successful ensure_spooling_space()");
3260 return true;
3261 }
3262 // If there's not enough expansion space available, give up.
3263 if (_virtual_space.uncommitted_size() < refill_size_bytes) {
3264 return false;
3265 }
3266 // Otherwise, we try expansion.
3267 expand(refill_size_bytes, MinHeapDeltaBytes,
3268 CMSExpansionCause::_allocate_par_spooling_space);
3269 // Now go around the loop and try alloc again;
3270 // A competing allocation might beat us to the expansion space,
3271 // so we may go around the loop again if allocation fails again.
3272 if (GCExpandToAllocateDelayMillis > 0) {
3273 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
3274 }
3275 }
3276 }
3280 void ConcurrentMarkSweepGeneration::shrink(size_t bytes) {
3281 assert_locked_or_safepoint(Heap_lock);
3282 size_t size = ReservedSpace::page_align_size_down(bytes);
3283 if (size > 0) {
3284 shrink_by(size);
3285 }
3286 }
3288 bool ConcurrentMarkSweepGeneration::grow_by(size_t bytes) {
3289 assert_locked_or_safepoint(Heap_lock);
3290 bool result = _virtual_space.expand_by(bytes);
3291 if (result) {
3292 HeapWord* old_end = _cmsSpace->end();
3293 size_t new_word_size =
3294 heap_word_size(_virtual_space.committed_size());
3295 MemRegion mr(_cmsSpace->bottom(), new_word_size);
3296 _bts->resize(new_word_size); // resize the block offset shared array
3297 Universe::heap()->barrier_set()->resize_covered_region(mr);
3298 // Hmmmm... why doesn't CFLS::set_end verify locking?
3299 // This is quite ugly; FIX ME XXX
3300 _cmsSpace->assert_locked();
3301 _cmsSpace->set_end((HeapWord*)_virtual_space.high());
3303 // update the space and generation capacity counters
3304 if (UsePerfData) {
3305 _space_counters->update_capacity();
3306 _gen_counters->update_all();
3307 }
3309 if (Verbose && PrintGC) {
3310 size_t new_mem_size = _virtual_space.committed_size();
3311 size_t old_mem_size = new_mem_size - bytes;
3312 gclog_or_tty->print_cr("Expanding %s from %ldK by %ldK to %ldK",
3313 name(), old_mem_size/K, bytes/K, new_mem_size/K);
3314 }
3315 }
3316 return result;
3317 }
3319 bool ConcurrentMarkSweepGeneration::grow_to_reserved() {
3320 assert_locked_or_safepoint(Heap_lock);
3321 bool success = true;
3322 const size_t remaining_bytes = _virtual_space.uncommitted_size();
3323 if (remaining_bytes > 0) {
3324 success = grow_by(remaining_bytes);
3325 DEBUG_ONLY(if (!success) warning("grow to reserved failed");)
3326 }
3327 return success;
3328 }
3330 void ConcurrentMarkSweepGeneration::shrink_by(size_t bytes) {
3331 assert_locked_or_safepoint(Heap_lock);
3332 assert_lock_strong(freelistLock());
3333 // XXX Fix when compaction is implemented.
3334 warning("Shrinking of CMS not yet implemented");
3335 return;
3336 }
3339 // Simple ctor/dtor wrapper for accounting & timer chores around concurrent
3340 // phases.
3341 class CMSPhaseAccounting: public StackObj {
3342 public:
3343 CMSPhaseAccounting(CMSCollector *collector,
3344 const char *phase,
3345 bool print_cr = true);
3346 ~CMSPhaseAccounting();
3348 private:
3349 CMSCollector *_collector;
3350 const char *_phase;
3351 elapsedTimer _wallclock;
3352 bool _print_cr;
3354 public:
3355 // Not MT-safe; so do not pass around these StackObj's
3356 // where they may be accessed by other threads.
3357 jlong wallclock_millis() {
3358 assert(_wallclock.is_active(), "Wall clock should not stop");
3359 _wallclock.stop(); // to record time
3360 jlong ret = _wallclock.milliseconds();
3361 _wallclock.start(); // restart
3362 return ret;
3363 }
3364 };
3366 CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector,
3367 const char *phase,
3368 bool print_cr) :
3369 _collector(collector), _phase(phase), _print_cr(print_cr) {
3371 if (PrintCMSStatistics != 0) {
3372 _collector->resetYields();
3373 }
3374 if (PrintGCDetails && PrintGCTimeStamps) {
3375 gclog_or_tty->date_stamp(PrintGCDateStamps);
3376 gclog_or_tty->stamp();
3377 gclog_or_tty->print_cr(": [%s-concurrent-%s-start]",
3378 _collector->cmsGen()->short_name(), _phase);
3379 }
3380 _collector->resetTimer();
3381 _wallclock.start();
3382 _collector->startTimer();
3383 }
3385 CMSPhaseAccounting::~CMSPhaseAccounting() {
3386 assert(_wallclock.is_active(), "Wall clock should not have stopped");
3387 _collector->stopTimer();
3388 _wallclock.stop();
3389 if (PrintGCDetails) {
3390 gclog_or_tty->date_stamp(PrintGCDateStamps);
3391 if (PrintGCTimeStamps) {
3392 gclog_or_tty->stamp();
3393 gclog_or_tty->print(": ");
3394 }
3395 gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]",
3396 _collector->cmsGen()->short_name(),
3397 _phase, _collector->timerValue(), _wallclock.seconds());
3398 if (_print_cr) {
3399 gclog_or_tty->print_cr("");
3400 }
3401 if (PrintCMSStatistics != 0) {
3402 gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase,
3403 _collector->yields());
3404 }
3405 }
3406 }
3408 // CMS work
3410 // Checkpoint the roots into this generation from outside
3411 // this generation. [Note this initial checkpoint need only
3412 // be approximate -- we'll do a catch up phase subsequently.]
3413 void CMSCollector::checkpointRootsInitial(bool asynch) {
3414 assert(_collectorState == InitialMarking, "Wrong collector state");
3415 check_correct_thread_executing();
3416 ReferenceProcessor* rp = ref_processor();
3417 SpecializationStats::clear();
3418 assert(_restart_addr == NULL, "Control point invariant");
3419 if (asynch) {
3420 // acquire locks for subsequent manipulations
3421 MutexLockerEx x(bitMapLock(),
3422 Mutex::_no_safepoint_check_flag);
3423 checkpointRootsInitialWork(asynch);
3424 rp->verify_no_references_recorded();
3425 rp->enable_discovery(); // enable ("weak") refs discovery
3426 _collectorState = Marking;
3427 } else {
3428 // (Weak) Refs discovery: this is controlled from genCollectedHeap::do_collection
3429 // which recognizes if we are a CMS generation, and doesn't try to turn on
3430 // discovery; verify that they aren't meddling.
3431 assert(!rp->discovery_is_atomic(),
3432 "incorrect setting of discovery predicate");
3433 assert(!rp->discovery_enabled(), "genCollectedHeap shouldn't control "
3434 "ref discovery for this generation kind");
3435 // already have locks
3436 checkpointRootsInitialWork(asynch);
3437 rp->enable_discovery(); // now enable ("weak") refs discovery
3438 _collectorState = Marking;
3439 }
3440 SpecializationStats::print();
3441 }
3443 void CMSCollector::checkpointRootsInitialWork(bool asynch) {
3444 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
3445 assert(_collectorState == InitialMarking, "just checking");
3447 // If there has not been a GC[n-1] since last GC[n] cycle completed,
3448 // precede our marking with a collection of all
3449 // younger generations to keep floating garbage to a minimum.
3450 // XXX: we won't do this for now -- it's an optimization to be done later.
3452 // already have locks
3453 assert_lock_strong(bitMapLock());
3454 assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
3456 // Setup the verification and class unloading state for this
3457 // CMS collection cycle.
3458 setup_cms_unloading_and_verification_state();
3460 NOT_PRODUCT(TraceTime t("\ncheckpointRootsInitialWork",
3461 PrintGCDetails && Verbose, true, gclog_or_tty);)
3462 if (UseAdaptiveSizePolicy) {
3463 size_policy()->checkpoint_roots_initial_begin();
3464 }
3466 // Reset all the PLAB chunk arrays if necessary.
3467 if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
3468 reset_survivor_plab_arrays();
3469 }
3471 ResourceMark rm;
3472 HandleMark hm;
3474 FalseClosure falseClosure;
3475 // In the case of a synchronous collection, we will elide the
3476 // remark step, so it's important to catch all the nmethod oops
3477 // in this step; hence the last argument to the constrcutor below.
3478 MarkRefsIntoClosure notOlder(_span, &_markBitMap, !asynch /* nmethods */);
3479 GenCollectedHeap* gch = GenCollectedHeap::heap();
3481 verify_work_stacks_empty();
3482 verify_overflow_empty();
3484 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
3485 // Update the saved marks which may affect the root scans.
3486 gch->save_marks();
3488 // weak reference processing has not started yet.
3489 ref_processor()->set_enqueuing_is_done(false);
3491 {
3492 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
3493 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
3494 gch->gen_process_strong_roots(_cmsGen->level(),
3495 true, // younger gens are roots
3496 true, // collecting perm gen
3497 SharedHeap::ScanningOption(roots_scanning_options()),
3498 NULL, ¬Older);
3499 }
3501 // Clear mod-union table; it will be dirtied in the prologue of
3502 // CMS generation per each younger generation collection.
3504 assert(_modUnionTable.isAllClear(),
3505 "Was cleared in most recent final checkpoint phase"
3506 " or no bits are set in the gc_prologue before the start of the next "
3507 "subsequent marking phase.");
3509 // Temporarily disabled, since pre/post-consumption closures don't
3510 // care about precleaned cards
3511 #if 0
3512 {
3513 MemRegion mr = MemRegion((HeapWord*)_virtual_space.low(),
3514 (HeapWord*)_virtual_space.high());
3515 _ct->ct_bs()->preclean_dirty_cards(mr);
3516 }
3517 #endif
3519 // Save the end of the used_region of the constituent generations
3520 // to be used to limit the extent of sweep in each generation.
3521 save_sweep_limits();
3522 if (UseAdaptiveSizePolicy) {
3523 size_policy()->checkpoint_roots_initial_end(gch->gc_cause());
3524 }
3525 verify_overflow_empty();
3526 }
3528 bool CMSCollector::markFromRoots(bool asynch) {
3529 // we might be tempted to assert that:
3530 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
3531 // "inconsistent argument?");
3532 // However that wouldn't be right, because it's possible that
3533 // a safepoint is indeed in progress as a younger generation
3534 // stop-the-world GC happens even as we mark in this generation.
3535 assert(_collectorState == Marking, "inconsistent state?");
3536 check_correct_thread_executing();
3537 verify_overflow_empty();
3539 bool res;
3540 if (asynch) {
3542 // Start the timers for adaptive size policy for the concurrent phases
3543 // Do it here so that the foreground MS can use the concurrent
3544 // timer since a foreground MS might has the sweep done concurrently
3545 // or STW.
3546 if (UseAdaptiveSizePolicy) {
3547 size_policy()->concurrent_marking_begin();
3548 }
3550 // Weak ref discovery note: We may be discovering weak
3551 // refs in this generation concurrent (but interleaved) with
3552 // weak ref discovery by a younger generation collector.
3554 CMSTokenSyncWithLocks ts(true, bitMapLock());
3555 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3556 CMSPhaseAccounting pa(this, "mark", !PrintGCDetails);
3557 res = markFromRootsWork(asynch);
3558 if (res) {
3559 _collectorState = Precleaning;
3560 } else { // We failed and a foreground collection wants to take over
3561 assert(_foregroundGCIsActive, "internal state inconsistency");
3562 assert(_restart_addr == NULL, "foreground will restart from scratch");
3563 if (PrintGCDetails) {
3564 gclog_or_tty->print_cr("bailing out to foreground collection");
3565 }
3566 }
3567 if (UseAdaptiveSizePolicy) {
3568 size_policy()->concurrent_marking_end();
3569 }
3570 } else {
3571 assert(SafepointSynchronize::is_at_safepoint(),
3572 "inconsistent with asynch == false");
3573 if (UseAdaptiveSizePolicy) {
3574 size_policy()->ms_collection_marking_begin();
3575 }
3576 // already have locks
3577 res = markFromRootsWork(asynch);
3578 _collectorState = FinalMarking;
3579 if (UseAdaptiveSizePolicy) {
3580 GenCollectedHeap* gch = GenCollectedHeap::heap();
3581 size_policy()->ms_collection_marking_end(gch->gc_cause());
3582 }
3583 }
3584 verify_overflow_empty();
3585 return res;
3586 }
3588 bool CMSCollector::markFromRootsWork(bool asynch) {
3589 // iterate over marked bits in bit map, doing a full scan and mark
3590 // from these roots using the following algorithm:
3591 // . if oop is to the right of the current scan pointer,
3592 // mark corresponding bit (we'll process it later)
3593 // . else (oop is to left of current scan pointer)
3594 // push oop on marking stack
3595 // . drain the marking stack
3597 // Note that when we do a marking step we need to hold the
3598 // bit map lock -- recall that direct allocation (by mutators)
3599 // and promotion (by younger generation collectors) is also
3600 // marking the bit map. [the so-called allocate live policy.]
3601 // Because the implementation of bit map marking is not
3602 // robust wrt simultaneous marking of bits in the same word,
3603 // we need to make sure that there is no such interference
3604 // between concurrent such updates.
3606 // already have locks
3607 assert_lock_strong(bitMapLock());
3609 // Clear the revisit stack, just in case there are any
3610 // obsolete contents from a short-circuited previous CMS cycle.
3611 _revisitStack.reset();
3612 verify_work_stacks_empty();
3613 verify_overflow_empty();
3614 assert(_revisitStack.isEmpty(), "tabula rasa");
3616 bool result = false;
3617 if (CMSConcurrentMTEnabled && ParallelCMSThreads > 0) {
3618 result = do_marking_mt(asynch);
3619 } else {
3620 result = do_marking_st(asynch);
3621 }
3622 return result;
3623 }
3625 // Forward decl
3626 class CMSConcMarkingTask;
3628 class CMSConcMarkingTerminator: public ParallelTaskTerminator {
3629 CMSCollector* _collector;
3630 CMSConcMarkingTask* _task;
3631 bool _yield;
3632 protected:
3633 virtual void yield();
3634 public:
3635 // "n_threads" is the number of threads to be terminated.
3636 // "queue_set" is a set of work queues of other threads.
3637 // "collector" is the CMS collector associated with this task terminator.
3638 // "yield" indicates whether we need the gang as a whole to yield.
3639 CMSConcMarkingTerminator(int n_threads, TaskQueueSetSuper* queue_set,
3640 CMSCollector* collector, bool yield) :
3641 ParallelTaskTerminator(n_threads, queue_set),
3642 _collector(collector),
3643 _yield(yield) { }
3645 void set_task(CMSConcMarkingTask* task) {
3646 _task = task;
3647 }
3648 };
3650 // MT Concurrent Marking Task
3651 class CMSConcMarkingTask: public YieldingFlexibleGangTask {
3652 CMSCollector* _collector;
3653 YieldingFlexibleWorkGang* _workers; // the whole gang
3654 int _n_workers; // requested/desired # workers
3655 bool _asynch;
3656 bool _result;
3657 CompactibleFreeListSpace* _cms_space;
3658 CompactibleFreeListSpace* _perm_space;
3659 HeapWord* _global_finger;
3661 // Exposed here for yielding support
3662 Mutex* const _bit_map_lock;
3664 // The per thread work queues, available here for stealing
3665 OopTaskQueueSet* _task_queues;
3666 CMSConcMarkingTerminator _term;
3668 public:
3669 CMSConcMarkingTask(CMSCollector* collector,
3670 CompactibleFreeListSpace* cms_space,
3671 CompactibleFreeListSpace* perm_space,
3672 bool asynch, int n_workers,
3673 YieldingFlexibleWorkGang* workers,
3674 OopTaskQueueSet* task_queues):
3675 YieldingFlexibleGangTask("Concurrent marking done multi-threaded"),
3676 _collector(collector),
3677 _cms_space(cms_space),
3678 _perm_space(perm_space),
3679 _asynch(asynch), _n_workers(n_workers), _result(true),
3680 _workers(workers), _task_queues(task_queues),
3681 _term(n_workers, task_queues, _collector, asynch),
3682 _bit_map_lock(collector->bitMapLock())
3683 {
3684 assert(n_workers <= workers->total_workers(),
3685 "Else termination won't work correctly today"); // XXX FIX ME!
3686 _requested_size = n_workers;
3687 _term.set_task(this);
3688 assert(_cms_space->bottom() < _perm_space->bottom(),
3689 "Finger incorrectly initialized below");
3690 _global_finger = _cms_space->bottom();
3691 }
3694 OopTaskQueueSet* task_queues() { return _task_queues; }
3696 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
3698 HeapWord** global_finger_addr() { return &_global_finger; }
3700 CMSConcMarkingTerminator* terminator() { return &_term; }
3702 void work(int i);
3704 virtual void coordinator_yield(); // stuff done by coordinator
3705 bool result() { return _result; }
3707 void reset(HeapWord* ra) {
3708 _term.reset_for_reuse();
3709 }
3711 static bool get_work_from_overflow_stack(CMSMarkStack* ovflw_stk,
3712 OopTaskQueue* work_q);
3714 private:
3715 void do_scan_and_mark(int i, CompactibleFreeListSpace* sp);
3716 void do_work_steal(int i);
3717 void bump_global_finger(HeapWord* f);
3718 };
3720 void CMSConcMarkingTerminator::yield() {
3721 if (ConcurrentMarkSweepThread::should_yield() &&
3722 !_collector->foregroundGCIsActive() &&
3723 _yield) {
3724 _task->yield();
3725 } else {
3726 ParallelTaskTerminator::yield();
3727 }
3728 }
3730 ////////////////////////////////////////////////////////////////
3731 // Concurrent Marking Algorithm Sketch
3732 ////////////////////////////////////////////////////////////////
3733 // Until all tasks exhausted (both spaces):
3734 // -- claim next available chunk
3735 // -- bump global finger via CAS
3736 // -- find first object that starts in this chunk
3737 // and start scanning bitmap from that position
3738 // -- scan marked objects for oops
3739 // -- CAS-mark target, and if successful:
3740 // . if target oop is above global finger (volatile read)
3741 // nothing to do
3742 // . if target oop is in chunk and above local finger
3743 // then nothing to do
3744 // . else push on work-queue
3745 // -- Deal with possible overflow issues:
3746 // . local work-queue overflow causes stuff to be pushed on
3747 // global (common) overflow queue
3748 // . always first empty local work queue
3749 // . then get a batch of oops from global work queue if any
3750 // . then do work stealing
3751 // -- When all tasks claimed (both spaces)
3752 // and local work queue empty,
3753 // then in a loop do:
3754 // . check global overflow stack; steal a batch of oops and trace
3755 // . try to steal from other threads oif GOS is empty
3756 // . if neither is available, offer termination
3757 // -- Terminate and return result
3758 //
3759 void CMSConcMarkingTask::work(int i) {
3760 elapsedTimer _timer;
3761 ResourceMark rm;
3762 HandleMark hm;
3764 DEBUG_ONLY(_collector->verify_overflow_empty();)
3766 // Before we begin work, our work queue should be empty
3767 assert(work_queue(i)->size() == 0, "Expected to be empty");
3768 // Scan the bitmap covering _cms_space, tracing through grey objects.
3769 _timer.start();
3770 do_scan_and_mark(i, _cms_space);
3771 _timer.stop();
3772 if (PrintCMSStatistics != 0) {
3773 gclog_or_tty->print_cr("Finished cms space scanning in %dth thread: %3.3f sec",
3774 i, _timer.seconds()); // XXX: need xxx/xxx type of notation, two timers
3775 }
3777 // ... do the same for the _perm_space
3778 _timer.reset();
3779 _timer.start();
3780 do_scan_and_mark(i, _perm_space);
3781 _timer.stop();
3782 if (PrintCMSStatistics != 0) {
3783 gclog_or_tty->print_cr("Finished perm space scanning in %dth thread: %3.3f sec",
3784 i, _timer.seconds()); // XXX: need xxx/xxx type of notation, two timers
3785 }
3787 // ... do work stealing
3788 _timer.reset();
3789 _timer.start();
3790 do_work_steal(i);
3791 _timer.stop();
3792 if (PrintCMSStatistics != 0) {
3793 gclog_or_tty->print_cr("Finished work stealing in %dth thread: %3.3f sec",
3794 i, _timer.seconds()); // XXX: need xxx/xxx type of notation, two timers
3795 }
3796 assert(_collector->_markStack.isEmpty(), "Should have been emptied");
3797 assert(work_queue(i)->size() == 0, "Should have been emptied");
3798 // Note that under the current task protocol, the
3799 // following assertion is true even of the spaces
3800 // expanded since the completion of the concurrent
3801 // marking. XXX This will likely change under a strict
3802 // ABORT semantics.
3803 assert(_global_finger > _cms_space->end() &&
3804 _global_finger >= _perm_space->end(),
3805 "All tasks have been completed");
3806 DEBUG_ONLY(_collector->verify_overflow_empty();)
3807 }
3809 void CMSConcMarkingTask::bump_global_finger(HeapWord* f) {
3810 HeapWord* read = _global_finger;
3811 HeapWord* cur = read;
3812 while (f > read) {
3813 cur = read;
3814 read = (HeapWord*) Atomic::cmpxchg_ptr(f, &_global_finger, cur);
3815 if (cur == read) {
3816 // our cas succeeded
3817 assert(_global_finger >= f, "protocol consistency");
3818 break;
3819 }
3820 }
3821 }
3823 // This is really inefficient, and should be redone by
3824 // using (not yet available) block-read and -write interfaces to the
3825 // stack and the work_queue. XXX FIX ME !!!
3826 bool CMSConcMarkingTask::get_work_from_overflow_stack(CMSMarkStack* ovflw_stk,
3827 OopTaskQueue* work_q) {
3828 // Fast lock-free check
3829 if (ovflw_stk->length() == 0) {
3830 return false;
3831 }
3832 assert(work_q->size() == 0, "Shouldn't steal");
3833 MutexLockerEx ml(ovflw_stk->par_lock(),
3834 Mutex::_no_safepoint_check_flag);
3835 // Grab up to 1/4 the size of the work queue
3836 size_t num = MIN2((size_t)work_q->max_elems()/4,
3837 (size_t)ParGCDesiredObjsFromOverflowList);
3838 num = MIN2(num, ovflw_stk->length());
3839 for (int i = (int) num; i > 0; i--) {
3840 oop cur = ovflw_stk->pop();
3841 assert(cur != NULL, "Counted wrong?");
3842 work_q->push(cur);
3843 }
3844 return num > 0;
3845 }
3847 void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
3848 SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
3849 int n_tasks = pst->n_tasks();
3850 // We allow that there may be no tasks to do here because
3851 // we are restarting after a stack overflow.
3852 assert(pst->valid() || n_tasks == 0, "Uninitializd use?");
3853 int nth_task = 0;
3855 HeapWord* start = sp->bottom();
3856 size_t chunk_size = sp->marking_task_size();
3857 while (!pst->is_task_claimed(/* reference */ nth_task)) {
3858 // Having claimed the nth task in this space,
3859 // compute the chunk that it corresponds to:
3860 MemRegion span = MemRegion(start + nth_task*chunk_size,
3861 start + (nth_task+1)*chunk_size);
3862 // Try and bump the global finger via a CAS;
3863 // note that we need to do the global finger bump
3864 // _before_ taking the intersection below, because
3865 // the task corresponding to that region will be
3866 // deemed done even if the used_region() expands
3867 // because of allocation -- as it almost certainly will
3868 // during start-up while the threads yield in the
3869 // closure below.
3870 HeapWord* finger = span.end();
3871 bump_global_finger(finger); // atomically
3872 // There are null tasks here corresponding to chunks
3873 // beyond the "top" address of the space.
3874 span = span.intersection(sp->used_region());
3875 if (!span.is_empty()) { // Non-null task
3876 // We want to skip the first object because
3877 // the protocol is to scan any object in its entirety
3878 // that _starts_ in this span; a fortiori, any
3879 // object starting in an earlier span is scanned
3880 // as part of an earlier claimed task.
3881 // Below we use the "careful" version of block_start
3882 // so we do not try to navigate uninitialized objects.
3883 HeapWord* prev_obj = sp->block_start_careful(span.start());
3884 // Below we use a variant of block_size that uses the
3885 // Printezis bits to avoid waiting for allocated
3886 // objects to become initialized/parsable.
3887 while (prev_obj < span.start()) {
3888 size_t sz = sp->block_size_no_stall(prev_obj, _collector);
3889 if (sz > 0) {
3890 prev_obj += sz;
3891 } else {
3892 // In this case we may end up doing a bit of redundant
3893 // scanning, but that appears unavoidable, short of
3894 // locking the free list locks; see bug 6324141.
3895 break;
3896 }
3897 }
3898 if (prev_obj < span.end()) {
3899 MemRegion my_span = MemRegion(prev_obj, span.end());
3900 // Do the marking work within a non-empty span --
3901 // the last argument to the constructor indicates whether the
3902 // iteration should be incremental with periodic yields.
3903 Par_MarkFromRootsClosure cl(this, _collector, my_span,
3904 &_collector->_markBitMap,
3905 work_queue(i),
3906 &_collector->_markStack,
3907 &_collector->_revisitStack,
3908 _asynch);
3909 _collector->_markBitMap.iterate(&cl, my_span.start(), my_span.end());
3910 } // else nothing to do for this task
3911 } // else nothing to do for this task
3912 }
3913 // We'd be tempted to assert here that since there are no
3914 // more tasks left to claim in this space, the global_finger
3915 // must exceed space->top() and a fortiori space->end(). However,
3916 // that would not quite be correct because the bumping of
3917 // global_finger occurs strictly after the claiming of a task,
3918 // so by the time we reach here the global finger may not yet
3919 // have been bumped up by the thread that claimed the last
3920 // task.
3921 pst->all_tasks_completed();
3922 }
3924 class Par_ConcMarkingClosure: public OopClosure {
3925 CMSCollector* _collector;
3926 MemRegion _span;
3927 CMSBitMap* _bit_map;
3928 CMSMarkStack* _overflow_stack;
3929 CMSMarkStack* _revisit_stack; // XXXXXX Check proper use
3930 OopTaskQueue* _work_queue;
3932 public:
3933 Par_ConcMarkingClosure(CMSCollector* collector, OopTaskQueue* work_queue,
3934 CMSBitMap* bit_map, CMSMarkStack* overflow_stack):
3935 _collector(collector),
3936 _span(_collector->_span),
3937 _work_queue(work_queue),
3938 _bit_map(bit_map),
3939 _overflow_stack(overflow_stack) { } // need to initialize revisit stack etc.
3941 void do_oop(oop* p);
3942 void trim_queue(size_t max);
3943 void handle_stack_overflow(HeapWord* lost);
3944 };
3946 // Grey object rescan during work stealing phase --
3947 // the salient assumption here is that stolen oops must
3948 // always be initialized, so we do not need to check for
3949 // uninitialized objects before scanning here.
3950 void Par_ConcMarkingClosure::do_oop(oop* p) {
3951 oop this_oop = *p;
3952 assert(this_oop->is_oop_or_null(),
3953 "expected an oop or NULL");
3954 HeapWord* addr = (HeapWord*)this_oop;
3955 // Check if oop points into the CMS generation
3956 // and is not marked
3957 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
3958 // a white object ...
3959 // If we manage to "claim" the object, by being the
3960 // first thread to mark it, then we push it on our
3961 // marking stack
3962 if (_bit_map->par_mark(addr)) { // ... now grey
3963 // push on work queue (grey set)
3964 bool simulate_overflow = false;
3965 NOT_PRODUCT(
3966 if (CMSMarkStackOverflowALot &&
3967 _collector->simulate_overflow()) {
3968 // simulate a stack overflow
3969 simulate_overflow = true;
3970 }
3971 )
3972 if (simulate_overflow ||
3973 !(_work_queue->push(this_oop) || _overflow_stack->par_push(this_oop))) {
3974 // stack overflow
3975 if (PrintCMSStatistics != 0) {
3976 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
3977 SIZE_FORMAT, _overflow_stack->capacity());
3978 }
3979 // We cannot assert that the overflow stack is full because
3980 // it may have been emptied since.
3981 assert(simulate_overflow ||
3982 _work_queue->size() == _work_queue->max_elems(),
3983 "Else push should have succeeded");
3984 handle_stack_overflow(addr);
3985 }
3986 } // Else, some other thread got there first
3987 }
3988 }
3990 void Par_ConcMarkingClosure::trim_queue(size_t max) {
3991 while (_work_queue->size() > max) {
3992 oop new_oop;
3993 if (_work_queue->pop_local(new_oop)) {
3994 assert(new_oop->is_oop(), "Should be an oop");
3995 assert(_bit_map->isMarked((HeapWord*)new_oop), "Grey object");
3996 assert(_span.contains((HeapWord*)new_oop), "Not in span");
3997 assert(new_oop->is_parsable(), "Should be parsable");
3998 new_oop->oop_iterate(this); // do_oop() above
3999 }
4000 }
4001 }
4003 // Upon stack overflow, we discard (part of) the stack,
4004 // remembering the least address amongst those discarded
4005 // in CMSCollector's _restart_address.
4006 void Par_ConcMarkingClosure::handle_stack_overflow(HeapWord* lost) {
4007 // We need to do this under a mutex to prevent other
4008 // workers from interfering with the expansion below.
4009 MutexLockerEx ml(_overflow_stack->par_lock(),
4010 Mutex::_no_safepoint_check_flag);
4011 // Remember the least grey address discarded
4012 HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost);
4013 _collector->lower_restart_addr(ra);
4014 _overflow_stack->reset(); // discard stack contents
4015 _overflow_stack->expand(); // expand the stack if possible
4016 }
4019 void CMSConcMarkingTask::do_work_steal(int i) {
4020 OopTaskQueue* work_q = work_queue(i);
4021 oop obj_to_scan;
4022 CMSBitMap* bm = &(_collector->_markBitMap);
4023 CMSMarkStack* ovflw = &(_collector->_markStack);
4024 int* seed = _collector->hash_seed(i);
4025 Par_ConcMarkingClosure cl(_collector, work_q, bm, ovflw);
4026 while (true) {
4027 cl.trim_queue(0);
4028 assert(work_q->size() == 0, "Should have been emptied above");
4029 if (get_work_from_overflow_stack(ovflw, work_q)) {
4030 // Can't assert below because the work obtained from the
4031 // overflow stack may already have been stolen from us.
4032 // assert(work_q->size() > 0, "Work from overflow stack");
4033 continue;
4034 } else if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
4035 assert(obj_to_scan->is_oop(), "Should be an oop");
4036 assert(bm->isMarked((HeapWord*)obj_to_scan), "Grey object");
4037 obj_to_scan->oop_iterate(&cl);
4038 } else if (terminator()->offer_termination()) {
4039 assert(work_q->size() == 0, "Impossible!");
4040 break;
4041 }
4042 }
4043 }
4045 // This is run by the CMS (coordinator) thread.
4046 void CMSConcMarkingTask::coordinator_yield() {
4047 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
4048 "CMS thread should hold CMS token");
4050 // First give up the locks, then yield, then re-lock
4051 // We should probably use a constructor/destructor idiom to
4052 // do this unlock/lock or modify the MutexUnlocker class to
4053 // serve our purpose. XXX
4054 assert_lock_strong(_bit_map_lock);
4055 _bit_map_lock->unlock();
4056 ConcurrentMarkSweepThread::desynchronize(true);
4057 ConcurrentMarkSweepThread::acknowledge_yield_request();
4058 _collector->stopTimer();
4059 if (PrintCMSStatistics != 0) {
4060 _collector->incrementYields();
4061 }
4062 _collector->icms_wait();
4064 // It is possible for whichever thread initiated the yield request
4065 // not to get a chance to wake up and take the bitmap lock between
4066 // this thread releasing it and reacquiring it. So, while the
4067 // should_yield() flag is on, let's sleep for a bit to give the
4068 // other thread a chance to wake up. The limit imposed on the number
4069 // of iterations is defensive, to avoid any unforseen circumstances
4070 // putting us into an infinite loop. Since it's always been this
4071 // (coordinator_yield()) method that was observed to cause the
4072 // problem, we are using a parameter (CMSCoordinatorYieldSleepCount)
4073 // which is by default non-zero. For the other seven methods that
4074 // also perform the yield operation, as are using a different
4075 // parameter (CMSYieldSleepCount) which is by default zero. This way we
4076 // can enable the sleeping for those methods too, if necessary.
4077 // See 6442774.
4078 //
4079 // We really need to reconsider the synchronization between the GC
4080 // thread and the yield-requesting threads in the future and we
4081 // should really use wait/notify, which is the recommended
4082 // way of doing this type of interaction. Additionally, we should
4083 // consolidate the eight methods that do the yield operation and they
4084 // are almost identical into one for better maintenability and
4085 // readability. See 6445193.
4086 //
4087 // Tony 2006.06.29
4088 for (unsigned i = 0; i < CMSCoordinatorYieldSleepCount &&
4089 ConcurrentMarkSweepThread::should_yield() &&
4090 !CMSCollector::foregroundGCIsActive(); ++i) {
4091 os::sleep(Thread::current(), 1, false);
4092 ConcurrentMarkSweepThread::acknowledge_yield_request();
4093 }
4095 ConcurrentMarkSweepThread::synchronize(true);
4096 _bit_map_lock->lock_without_safepoint_check();
4097 _collector->startTimer();
4098 }
4100 bool CMSCollector::do_marking_mt(bool asynch) {
4101 assert(ParallelCMSThreads > 0 && conc_workers() != NULL, "precondition");
4102 // In the future this would be determined ergonomically, based
4103 // on #cpu's, # active mutator threads (and load), and mutation rate.
4104 int num_workers = ParallelCMSThreads;
4106 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
4107 CompactibleFreeListSpace* perm_space = _permGen->cmsSpace();
4109 CMSConcMarkingTask tsk(this, cms_space, perm_space,
4110 asynch, num_workers /* number requested XXX */,
4111 conc_workers(), task_queues());
4113 // Since the actual number of workers we get may be different
4114 // from the number we requested above, do we need to do anything different
4115 // below? In particular, may be we need to subclass the SequantialSubTasksDone
4116 // class?? XXX
4117 cms_space ->initialize_sequential_subtasks_for_marking(num_workers);
4118 perm_space->initialize_sequential_subtasks_for_marking(num_workers);
4120 // Refs discovery is already non-atomic.
4121 assert(!ref_processor()->discovery_is_atomic(), "Should be non-atomic");
4122 // Mutate the Refs discovery so it is MT during the
4123 // multi-threaded marking phase.
4124 ReferenceProcessorMTMutator mt(ref_processor(), num_workers > 1);
4126 conc_workers()->start_task(&tsk);
4127 while (tsk.yielded()) {
4128 tsk.coordinator_yield();
4129 conc_workers()->continue_task(&tsk);
4130 }
4131 // If the task was aborted, _restart_addr will be non-NULL
4132 assert(tsk.completed() || _restart_addr != NULL, "Inconsistency");
4133 while (_restart_addr != NULL) {
4134 // XXX For now we do not make use of ABORTED state and have not
4135 // yet implemented the right abort semantics (even in the original
4136 // single-threaded CMS case). That needs some more investigation
4137 // and is deferred for now; see CR# TBF. 07252005YSR. XXX
4138 assert(!CMSAbortSemantics || tsk.aborted(), "Inconsistency");
4139 // If _restart_addr is non-NULL, a marking stack overflow
4140 // occured; we need to do a fresh marking iteration from the
4141 // indicated restart address.
4142 if (_foregroundGCIsActive && asynch) {
4143 // We may be running into repeated stack overflows, having
4144 // reached the limit of the stack size, while making very
4145 // slow forward progress. It may be best to bail out and
4146 // let the foreground collector do its job.
4147 // Clear _restart_addr, so that foreground GC
4148 // works from scratch. This avoids the headache of
4149 // a "rescan" which would otherwise be needed because
4150 // of the dirty mod union table & card table.
4151 _restart_addr = NULL;
4152 return false;
4153 }
4154 // Adjust the task to restart from _restart_addr
4155 tsk.reset(_restart_addr);
4156 cms_space ->initialize_sequential_subtasks_for_marking(num_workers,
4157 _restart_addr);
4158 perm_space->initialize_sequential_subtasks_for_marking(num_workers,
4159 _restart_addr);
4160 _restart_addr = NULL;
4161 // Get the workers going again
4162 conc_workers()->start_task(&tsk);
4163 while (tsk.yielded()) {
4164 tsk.coordinator_yield();
4165 conc_workers()->continue_task(&tsk);
4166 }
4167 }
4168 assert(tsk.completed(), "Inconsistency");
4169 assert(tsk.result() == true, "Inconsistency");
4170 return true;
4171 }
4173 bool CMSCollector::do_marking_st(bool asynch) {
4174 ResourceMark rm;
4175 HandleMark hm;
4177 MarkFromRootsClosure markFromRootsClosure(this, _span, &_markBitMap,
4178 &_markStack, &_revisitStack, CMSYield && asynch);
4179 // the last argument to iterate indicates whether the iteration
4180 // should be incremental with periodic yields.
4181 _markBitMap.iterate(&markFromRootsClosure);
4182 // If _restart_addr is non-NULL, a marking stack overflow
4183 // occured; we need to do a fresh iteration from the
4184 // indicated restart address.
4185 while (_restart_addr != NULL) {
4186 if (_foregroundGCIsActive && asynch) {
4187 // We may be running into repeated stack overflows, having
4188 // reached the limit of the stack size, while making very
4189 // slow forward progress. It may be best to bail out and
4190 // let the foreground collector do its job.
4191 // Clear _restart_addr, so that foreground GC
4192 // works from scratch. This avoids the headache of
4193 // a "rescan" which would otherwise be needed because
4194 // of the dirty mod union table & card table.
4195 _restart_addr = NULL;
4196 return false; // indicating failure to complete marking
4197 }
4198 // Deal with stack overflow:
4199 // we restart marking from _restart_addr
4200 HeapWord* ra = _restart_addr;
4201 markFromRootsClosure.reset(ra);
4202 _restart_addr = NULL;
4203 _markBitMap.iterate(&markFromRootsClosure, ra, _span.end());
4204 }
4205 return true;
4206 }
4208 void CMSCollector::preclean() {
4209 check_correct_thread_executing();
4210 assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
4211 verify_work_stacks_empty();
4212 verify_overflow_empty();
4213 _abort_preclean = false;
4214 if (CMSPrecleaningEnabled) {
4215 _eden_chunk_index = 0;
4216 size_t used = get_eden_used();
4217 size_t capacity = get_eden_capacity();
4218 // Don't start sampling unless we will get sufficiently
4219 // many samples.
4220 if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
4221 * CMSScheduleRemarkEdenPenetration)) {
4222 _start_sampling = true;
4223 } else {
4224 _start_sampling = false;
4225 }
4226 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
4227 CMSPhaseAccounting pa(this, "preclean", !PrintGCDetails);
4228 preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
4229 }
4230 CMSTokenSync x(true); // is cms thread
4231 if (CMSPrecleaningEnabled) {
4232 sample_eden();
4233 _collectorState = AbortablePreclean;
4234 } else {
4235 _collectorState = FinalMarking;
4236 }
4237 verify_work_stacks_empty();
4238 verify_overflow_empty();
4239 }
4241 // Try and schedule the remark such that young gen
4242 // occupancy is CMSScheduleRemarkEdenPenetration %.
4243 void CMSCollector::abortable_preclean() {
4244 check_correct_thread_executing();
4245 assert(CMSPrecleaningEnabled, "Inconsistent control state");
4246 assert(_collectorState == AbortablePreclean, "Inconsistent control state");
4248 // If Eden's current occupancy is below this threshold,
4249 // immediately schedule the remark; else preclean
4250 // past the next scavenge in an effort to
4251 // schedule the pause as described avove. By choosing
4252 // CMSScheduleRemarkEdenSizeThreshold >= max eden size
4253 // we will never do an actual abortable preclean cycle.
4254 if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) {
4255 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
4256 CMSPhaseAccounting pa(this, "abortable-preclean", !PrintGCDetails);
4257 // We need more smarts in the abortable preclean
4258 // loop below to deal with cases where allocation
4259 // in young gen is very very slow, and our precleaning
4260 // is running a losing race against a horde of
4261 // mutators intent on flooding us with CMS updates
4262 // (dirty cards).
4263 // One, admittedly dumb, strategy is to give up
4264 // after a certain number of abortable precleaning loops
4265 // or after a certain maximum time. We want to make
4266 // this smarter in the next iteration.
4267 // XXX FIX ME!!! YSR
4268 size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0;
4269 while (!(should_abort_preclean() ||
4270 ConcurrentMarkSweepThread::should_terminate())) {
4271 workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2);
4272 cumworkdone += workdone;
4273 loops++;
4274 // Voluntarily terminate abortable preclean phase if we have
4275 // been at it for too long.
4276 if ((CMSMaxAbortablePrecleanLoops != 0) &&
4277 loops >= CMSMaxAbortablePrecleanLoops) {
4278 if (PrintGCDetails) {
4279 gclog_or_tty->print(" CMS: abort preclean due to loops ");
4280 }
4281 break;
4282 }
4283 if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) {
4284 if (PrintGCDetails) {
4285 gclog_or_tty->print(" CMS: abort preclean due to time ");
4286 }
4287 break;
4288 }
4289 // If we are doing little work each iteration, we should
4290 // take a short break.
4291 if (workdone < CMSAbortablePrecleanMinWorkPerIteration) {
4292 // Sleep for some time, waiting for work to accumulate
4293 stopTimer();
4294 cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis);
4295 startTimer();
4296 waited++;
4297 }
4298 }
4299 if (PrintCMSStatistics > 0) {
4300 gclog_or_tty->print(" [%d iterations, %d waits, %d cards)] ",
4301 loops, waited, cumworkdone);
4302 }
4303 }
4304 CMSTokenSync x(true); // is cms thread
4305 if (_collectorState != Idling) {
4306 assert(_collectorState == AbortablePreclean,
4307 "Spontaneous state transition?");
4308 _collectorState = FinalMarking;
4309 } // Else, a foreground collection completed this CMS cycle.
4310 return;
4311 }
4313 // Respond to an Eden sampling opportunity
4314 void CMSCollector::sample_eden() {
4315 // Make sure a young gc cannot sneak in between our
4316 // reading and recording of a sample.
4317 assert(Thread::current()->is_ConcurrentGC_thread(),
4318 "Only the cms thread may collect Eden samples");
4319 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
4320 "Should collect samples while holding CMS token");
4321 if (!_start_sampling) {
4322 return;
4323 }
4324 if (_eden_chunk_array) {
4325 if (_eden_chunk_index < _eden_chunk_capacity) {
4326 _eden_chunk_array[_eden_chunk_index] = *_top_addr; // take sample
4327 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
4328 "Unexpected state of Eden");
4329 // We'd like to check that what we just sampled is an oop-start address;
4330 // however, we cannot do that here since the object may not yet have been
4331 // initialized. So we'll instead do the check when we _use_ this sample
4332 // later.
4333 if (_eden_chunk_index == 0 ||
4334 (pointer_delta(_eden_chunk_array[_eden_chunk_index],
4335 _eden_chunk_array[_eden_chunk_index-1])
4336 >= CMSSamplingGrain)) {
4337 _eden_chunk_index++; // commit sample
4338 }
4339 }
4340 }
4341 if ((_collectorState == AbortablePreclean) && !_abort_preclean) {
4342 size_t used = get_eden_used();
4343 size_t capacity = get_eden_capacity();
4344 assert(used <= capacity, "Unexpected state of Eden");
4345 if (used > (capacity/100 * CMSScheduleRemarkEdenPenetration)) {
4346 _abort_preclean = true;
4347 }
4348 }
4349 }
4352 size_t CMSCollector::preclean_work(bool clean_refs, bool clean_survivor) {
4353 assert(_collectorState == Precleaning ||
4354 _collectorState == AbortablePreclean, "incorrect state");
4355 ResourceMark rm;
4356 HandleMark hm;
4357 // Do one pass of scrubbing the discovered reference lists
4358 // to remove any reference objects with strongly-reachable
4359 // referents.
4360 if (clean_refs) {
4361 ReferenceProcessor* rp = ref_processor();
4362 CMSPrecleanRefsYieldClosure yield_cl(this);
4363 assert(rp->span().equals(_span), "Spans should be equal");
4364 CMSKeepAliveClosure keep_alive(this, _span, &_markBitMap,
4365 &_markStack);
4366 CMSDrainMarkingStackClosure complete_trace(this,
4367 _span, &_markBitMap, &_markStack,
4368 &keep_alive);
4370 // We don't want this step to interfere with a young
4371 // collection because we don't want to take CPU
4372 // or memory bandwidth away from the young GC threads
4373 // (which may be as many as there are CPUs).
4374 // Note that we don't need to protect ourselves from
4375 // interference with mutators because they can't
4376 // manipulate the discovered reference lists nor affect
4377 // the computed reachability of the referents, the
4378 // only properties manipulated by the precleaning
4379 // of these reference lists.
4380 stopTimer();
4381 CMSTokenSyncWithLocks x(true /* is cms thread */,
4382 bitMapLock());
4383 startTimer();
4384 sample_eden();
4385 // The following will yield to allow foreground
4386 // collection to proceed promptly. XXX YSR:
4387 // The code in this method may need further
4388 // tweaking for better performance and some restructuring
4389 // for cleaner interfaces.
4390 rp->preclean_discovered_references(
4391 rp->is_alive_non_header(), &keep_alive, &complete_trace,
4392 &yield_cl);
4393 }
4395 if (clean_survivor) { // preclean the active survivor space(s)
4396 assert(_young_gen->kind() == Generation::DefNew ||
4397 _young_gen->kind() == Generation::ParNew ||
4398 _young_gen->kind() == Generation::ASParNew,
4399 "incorrect type for cast");
4400 DefNewGeneration* dng = (DefNewGeneration*)_young_gen;
4401 PushAndMarkClosure pam_cl(this, _span, ref_processor(),
4402 &_markBitMap, &_modUnionTable,
4403 &_markStack, &_revisitStack,
4404 true /* precleaning phase */);
4405 stopTimer();
4406 CMSTokenSyncWithLocks ts(true /* is cms thread */,
4407 bitMapLock());
4408 startTimer();
4409 unsigned int before_count =
4410 GenCollectedHeap::heap()->total_collections();
4411 SurvivorSpacePrecleanClosure
4412 sss_cl(this, _span, &_markBitMap, &_markStack,
4413 &pam_cl, before_count, CMSYield);
4414 dng->from()->object_iterate_careful(&sss_cl);
4415 dng->to()->object_iterate_careful(&sss_cl);
4416 }
4417 MarkRefsIntoAndScanClosure
4418 mrias_cl(_span, ref_processor(), &_markBitMap, &_modUnionTable,
4419 &_markStack, &_revisitStack, this, CMSYield,
4420 true /* precleaning phase */);
4421 // CAUTION: The following closure has persistent state that may need to
4422 // be reset upon a decrease in the sequence of addresses it
4423 // processes.
4424 ScanMarkedObjectsAgainCarefullyClosure
4425 smoac_cl(this, _span,
4426 &_markBitMap, &_markStack, &_revisitStack, &mrias_cl, CMSYield);
4428 // Preclean dirty cards in ModUnionTable and CardTable using
4429 // appropriate convergence criterion;
4430 // repeat CMSPrecleanIter times unless we find that
4431 // we are losing.
4432 assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large");
4433 assert(CMSPrecleanNumerator < CMSPrecleanDenominator,
4434 "Bad convergence multiplier");
4435 assert(CMSPrecleanThreshold >= 100,
4436 "Unreasonably low CMSPrecleanThreshold");
4438 size_t numIter, cumNumCards, lastNumCards, curNumCards;
4439 for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0;
4440 numIter < CMSPrecleanIter;
4441 numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) {
4442 curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl);
4443 if (CMSPermGenPrecleaningEnabled) {
4444 curNumCards += preclean_mod_union_table(_permGen, &smoac_cl);
4445 }
4446 if (Verbose && PrintGCDetails) {
4447 gclog_or_tty->print(" (modUnionTable: %d cards)", curNumCards);
4448 }
4449 // Either there are very few dirty cards, so re-mark
4450 // pause will be small anyway, or our pre-cleaning isn't
4451 // that much faster than the rate at which cards are being
4452 // dirtied, so we might as well stop and re-mark since
4453 // precleaning won't improve our re-mark time by much.
4454 if (curNumCards <= CMSPrecleanThreshold ||
4455 (numIter > 0 &&
4456 (curNumCards * CMSPrecleanDenominator >
4457 lastNumCards * CMSPrecleanNumerator))) {
4458 numIter++;
4459 cumNumCards += curNumCards;
4460 break;
4461 }
4462 }
4463 curNumCards = preclean_card_table(_cmsGen, &smoac_cl);
4464 if (CMSPermGenPrecleaningEnabled) {
4465 curNumCards += preclean_card_table(_permGen, &smoac_cl);
4466 }
4467 cumNumCards += curNumCards;
4468 if (PrintGCDetails && PrintCMSStatistics != 0) {
4469 gclog_or_tty->print_cr(" (cardTable: %d cards, re-scanned %d cards, %d iterations)",
4470 curNumCards, cumNumCards, numIter);
4471 }
4472 return cumNumCards; // as a measure of useful work done
4473 }
4475 // PRECLEANING NOTES:
4476 // Precleaning involves:
4477 // . reading the bits of the modUnionTable and clearing the set bits.
4478 // . For the cards corresponding to the set bits, we scan the
4479 // objects on those cards. This means we need the free_list_lock
4480 // so that we can safely iterate over the CMS space when scanning
4481 // for oops.
4482 // . When we scan the objects, we'll be both reading and setting
4483 // marks in the marking bit map, so we'll need the marking bit map.
4484 // . For protecting _collector_state transitions, we take the CGC_lock.
4485 // Note that any races in the reading of of card table entries by the
4486 // CMS thread on the one hand and the clearing of those entries by the
4487 // VM thread or the setting of those entries by the mutator threads on the
4488 // other are quite benign. However, for efficiency it makes sense to keep
4489 // the VM thread from racing with the CMS thread while the latter is
4490 // dirty card info to the modUnionTable. We therefore also use the
4491 // CGC_lock to protect the reading of the card table and the mod union
4492 // table by the CM thread.
4493 // . We run concurrently with mutator updates, so scanning
4494 // needs to be done carefully -- we should not try to scan
4495 // potentially uninitialized objects.
4496 //
4497 // Locking strategy: While holding the CGC_lock, we scan over and
4498 // reset a maximal dirty range of the mod union / card tables, then lock
4499 // the free_list_lock and bitmap lock to do a full marking, then
4500 // release these locks; and repeat the cycle. This allows for a
4501 // certain amount of fairness in the sharing of these locks between
4502 // the CMS collector on the one hand, and the VM thread and the
4503 // mutators on the other.
4505 // NOTE: preclean_mod_union_table() and preclean_card_table()
4506 // further below are largely identical; if you need to modify
4507 // one of these methods, please check the other method too.
4509 size_t CMSCollector::preclean_mod_union_table(
4510 ConcurrentMarkSweepGeneration* gen,
4511 ScanMarkedObjectsAgainCarefullyClosure* cl) {
4512 verify_work_stacks_empty();
4513 verify_overflow_empty();
4515 // strategy: starting with the first card, accumulate contiguous
4516 // ranges of dirty cards; clear these cards, then scan the region
4517 // covered by these cards.
4519 // Since all of the MUT is committed ahead, we can just use
4520 // that, in case the generations expand while we are precleaning.
4521 // It might also be fine to just use the committed part of the
4522 // generation, but we might potentially miss cards when the
4523 // generation is rapidly expanding while we are in the midst
4524 // of precleaning.
4525 HeapWord* startAddr = gen->reserved().start();
4526 HeapWord* endAddr = gen->reserved().end();
4528 cl->setFreelistLock(gen->freelistLock()); // needed for yielding
4530 size_t numDirtyCards, cumNumDirtyCards;
4531 HeapWord *nextAddr, *lastAddr;
4532 for (cumNumDirtyCards = numDirtyCards = 0,
4533 nextAddr = lastAddr = startAddr;
4534 nextAddr < endAddr;
4535 nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
4537 ResourceMark rm;
4538 HandleMark hm;
4540 MemRegion dirtyRegion;
4541 {
4542 stopTimer();
4543 CMSTokenSync ts(true);
4544 startTimer();
4545 sample_eden();
4546 // Get dirty region starting at nextOffset (inclusive),
4547 // simultaneously clearing it.
4548 dirtyRegion =
4549 _modUnionTable.getAndClearMarkedRegion(nextAddr, endAddr);
4550 assert(dirtyRegion.start() >= nextAddr,
4551 "returned region inconsistent?");
4552 }
4553 // Remember where the next search should begin.
4554 // The returned region (if non-empty) is a right open interval,
4555 // so lastOffset is obtained from the right end of that
4556 // interval.
4557 lastAddr = dirtyRegion.end();
4558 // Should do something more transparent and less hacky XXX
4559 numDirtyCards =
4560 _modUnionTable.heapWordDiffToOffsetDiff(dirtyRegion.word_size());
4562 // We'll scan the cards in the dirty region (with periodic
4563 // yields for foreground GC as needed).
4564 if (!dirtyRegion.is_empty()) {
4565 assert(numDirtyCards > 0, "consistency check");
4566 HeapWord* stop_point = NULL;
4567 {
4568 stopTimer();
4569 CMSTokenSyncWithLocks ts(true, gen->freelistLock(),
4570 bitMapLock());
4571 startTimer();
4572 verify_work_stacks_empty();
4573 verify_overflow_empty();
4574 sample_eden();
4575 stop_point =
4576 gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
4577 }
4578 if (stop_point != NULL) {
4579 // The careful iteration stopped early either because it found an
4580 // uninitialized object, or because we were in the midst of an
4581 // "abortable preclean", which should now be aborted. Redirty
4582 // the bits corresponding to the partially-scanned or unscanned
4583 // cards. We'll either restart at the next block boundary or
4584 // abort the preclean.
4585 assert((CMSPermGenPrecleaningEnabled && (gen == _permGen)) ||
4586 (_collectorState == AbortablePreclean && should_abort_preclean()),
4587 "Unparsable objects should only be in perm gen.");
4589 stopTimer();
4590 CMSTokenSyncWithLocks ts(true, bitMapLock());
4591 startTimer();
4592 _modUnionTable.mark_range(MemRegion(stop_point, dirtyRegion.end()));
4593 if (should_abort_preclean()) {
4594 break; // out of preclean loop
4595 } else {
4596 // Compute the next address at which preclean should pick up;
4597 // might need bitMapLock in order to read P-bits.
4598 lastAddr = next_card_start_after_block(stop_point);
4599 }
4600 }
4601 } else {
4602 assert(lastAddr == endAddr, "consistency check");
4603 assert(numDirtyCards == 0, "consistency check");
4604 break;
4605 }
4606 }
4607 verify_work_stacks_empty();
4608 verify_overflow_empty();
4609 return cumNumDirtyCards;
4610 }
4612 // NOTE: preclean_mod_union_table() above and preclean_card_table()
4613 // below are largely identical; if you need to modify
4614 // one of these methods, please check the other method too.
4616 size_t CMSCollector::preclean_card_table(ConcurrentMarkSweepGeneration* gen,
4617 ScanMarkedObjectsAgainCarefullyClosure* cl) {
4618 // strategy: it's similar to precleamModUnionTable above, in that
4619 // we accumulate contiguous ranges of dirty cards, mark these cards
4620 // precleaned, then scan the region covered by these cards.
4621 HeapWord* endAddr = (HeapWord*)(gen->_virtual_space.high());
4622 HeapWord* startAddr = (HeapWord*)(gen->_virtual_space.low());
4624 cl->setFreelistLock(gen->freelistLock()); // needed for yielding
4626 size_t numDirtyCards, cumNumDirtyCards;
4627 HeapWord *lastAddr, *nextAddr;
4629 for (cumNumDirtyCards = numDirtyCards = 0,
4630 nextAddr = lastAddr = startAddr;
4631 nextAddr < endAddr;
4632 nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
4634 ResourceMark rm;
4635 HandleMark hm;
4637 MemRegion dirtyRegion;
4638 {
4639 // See comments in "Precleaning notes" above on why we
4640 // do this locking. XXX Could the locking overheads be
4641 // too high when dirty cards are sparse? [I don't think so.]
4642 stopTimer();
4643 CMSTokenSync x(true); // is cms thread
4644 startTimer();
4645 sample_eden();
4646 // Get and clear dirty region from card table
4647 dirtyRegion = _ct->ct_bs()->dirty_card_range_after_preclean(
4648 MemRegion(nextAddr, endAddr));
4649 assert(dirtyRegion.start() >= nextAddr,
4650 "returned region inconsistent?");
4651 }
4652 lastAddr = dirtyRegion.end();
4653 numDirtyCards =
4654 dirtyRegion.word_size()/CardTableModRefBS::card_size_in_words;
4656 if (!dirtyRegion.is_empty()) {
4657 stopTimer();
4658 CMSTokenSyncWithLocks ts(true, gen->freelistLock(), bitMapLock());
4659 startTimer();
4660 sample_eden();
4661 verify_work_stacks_empty();
4662 verify_overflow_empty();
4663 HeapWord* stop_point =
4664 gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
4665 if (stop_point != NULL) {
4666 // The careful iteration stopped early because it found an
4667 // uninitialized object. Redirty the bits corresponding to the
4668 // partially-scanned or unscanned cards, and start again at the
4669 // next block boundary.
4670 assert(CMSPermGenPrecleaningEnabled ||
4671 (_collectorState == AbortablePreclean && should_abort_preclean()),
4672 "Unparsable objects should only be in perm gen.");
4673 _ct->ct_bs()->invalidate(MemRegion(stop_point, dirtyRegion.end()));
4674 if (should_abort_preclean()) {
4675 break; // out of preclean loop
4676 } else {
4677 // Compute the next address at which preclean should pick up.
4678 lastAddr = next_card_start_after_block(stop_point);
4679 }
4680 }
4681 } else {
4682 break;
4683 }
4684 }
4685 verify_work_stacks_empty();
4686 verify_overflow_empty();
4687 return cumNumDirtyCards;
4688 }
4690 void CMSCollector::checkpointRootsFinal(bool asynch,
4691 bool clear_all_soft_refs, bool init_mark_was_synchronous) {
4692 assert(_collectorState == FinalMarking, "incorrect state transition?");
4693 check_correct_thread_executing();
4694 // world is stopped at this checkpoint
4695 assert(SafepointSynchronize::is_at_safepoint(),
4696 "world should be stopped");
4697 verify_work_stacks_empty();
4698 verify_overflow_empty();
4700 SpecializationStats::clear();
4701 if (PrintGCDetails) {
4702 gclog_or_tty->print("[YG occupancy: "SIZE_FORMAT" K ("SIZE_FORMAT" K)]",
4703 _young_gen->used() / K,
4704 _young_gen->capacity() / K);
4705 }
4706 if (asynch) {
4707 if (CMSScavengeBeforeRemark) {
4708 GenCollectedHeap* gch = GenCollectedHeap::heap();
4709 // Temporarily set flag to false, GCH->do_collection will
4710 // expect it to be false and set to true
4711 FlagSetting fl(gch->_is_gc_active, false);
4712 NOT_PRODUCT(TraceTime t("Scavenge-Before-Remark",
4713 PrintGCDetails && Verbose, true, gclog_or_tty);)
4714 int level = _cmsGen->level() - 1;
4715 if (level >= 0) {
4716 gch->do_collection(true, // full (i.e. force, see below)
4717 false, // !clear_all_soft_refs
4718 0, // size
4719 false, // is_tlab
4720 level // max_level
4721 );
4722 }
4723 }
4724 FreelistLocker x(this);
4725 MutexLockerEx y(bitMapLock(),
4726 Mutex::_no_safepoint_check_flag);
4727 assert(!init_mark_was_synchronous, "but that's impossible!");
4728 checkpointRootsFinalWork(asynch, clear_all_soft_refs, false);
4729 } else {
4730 // already have all the locks
4731 checkpointRootsFinalWork(asynch, clear_all_soft_refs,
4732 init_mark_was_synchronous);
4733 }
4734 verify_work_stacks_empty();
4735 verify_overflow_empty();
4736 SpecializationStats::print();
4737 }
4739 void CMSCollector::checkpointRootsFinalWork(bool asynch,
4740 bool clear_all_soft_refs, bool init_mark_was_synchronous) {
4742 NOT_PRODUCT(TraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, gclog_or_tty);)
4744 assert(haveFreelistLocks(), "must have free list locks");
4745 assert_lock_strong(bitMapLock());
4747 if (UseAdaptiveSizePolicy) {
4748 size_policy()->checkpoint_roots_final_begin();
4749 }
4751 ResourceMark rm;
4752 HandleMark hm;
4754 GenCollectedHeap* gch = GenCollectedHeap::heap();
4756 if (should_unload_classes()) {
4757 CodeCache::gc_prologue();
4758 }
4759 assert(haveFreelistLocks(), "must have free list locks");
4760 assert_lock_strong(bitMapLock());
4762 if (!init_mark_was_synchronous) {
4763 // We might assume that we need not fill TLAB's when
4764 // CMSScavengeBeforeRemark is set, because we may have just done
4765 // a scavenge which would have filled all TLAB's -- and besides
4766 // Eden would be empty. This however may not always be the case --
4767 // for instance although we asked for a scavenge, it may not have
4768 // happened because of a JNI critical section. We probably need
4769 // a policy for deciding whether we can in that case wait until
4770 // the critical section releases and then do the remark following
4771 // the scavenge, and skip it here. In the absence of that policy,
4772 // or of an indication of whether the scavenge did indeed occur,
4773 // we cannot rely on TLAB's having been filled and must do
4774 // so here just in case a scavenge did not happen.
4775 gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
4776 // Update the saved marks which may affect the root scans.
4777 gch->save_marks();
4779 {
4780 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
4782 // Note on the role of the mod union table:
4783 // Since the marker in "markFromRoots" marks concurrently with
4784 // mutators, it is possible for some reachable objects not to have been
4785 // scanned. For instance, an only reference to an object A was
4786 // placed in object B after the marker scanned B. Unless B is rescanned,
4787 // A would be collected. Such updates to references in marked objects
4788 // are detected via the mod union table which is the set of all cards
4789 // dirtied since the first checkpoint in this GC cycle and prior to
4790 // the most recent young generation GC, minus those cleaned up by the
4791 // concurrent precleaning.
4792 if (CMSParallelRemarkEnabled && ParallelGCThreads > 0) {
4793 TraceTime t("Rescan (parallel) ", PrintGCDetails, false, gclog_or_tty);
4794 do_remark_parallel();
4795 } else {
4796 TraceTime t("Rescan (non-parallel) ", PrintGCDetails, false,
4797 gclog_or_tty);
4798 do_remark_non_parallel();
4799 }
4800 }
4801 } else {
4802 assert(!asynch, "Can't have init_mark_was_synchronous in asynch mode");
4803 // The initial mark was stop-world, so there's no rescanning to
4804 // do; go straight on to the next step below.
4805 }
4806 verify_work_stacks_empty();
4807 verify_overflow_empty();
4809 {
4810 NOT_PRODUCT(TraceTime ts("refProcessingWork", PrintGCDetails, false, gclog_or_tty);)
4811 refProcessingWork(asynch, clear_all_soft_refs);
4812 }
4813 verify_work_stacks_empty();
4814 verify_overflow_empty();
4816 if (should_unload_classes()) {
4817 CodeCache::gc_epilogue();
4818 }
4820 // If we encountered any (marking stack / work queue) overflow
4821 // events during the current CMS cycle, take appropriate
4822 // remedial measures, where possible, so as to try and avoid
4823 // recurrence of that condition.
4824 assert(_markStack.isEmpty(), "No grey objects");
4825 size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
4826 _ser_kac_ovflw;
4827 if (ser_ovflw > 0) {
4828 if (PrintCMSStatistics != 0) {
4829 gclog_or_tty->print_cr("Marking stack overflow (benign) "
4830 "(pmc_pc="SIZE_FORMAT", pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT")",
4831 _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw,
4832 _ser_kac_ovflw);
4833 }
4834 _markStack.expand();
4835 _ser_pmc_remark_ovflw = 0;
4836 _ser_pmc_preclean_ovflw = 0;
4837 _ser_kac_ovflw = 0;
4838 }
4839 if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
4840 if (PrintCMSStatistics != 0) {
4841 gclog_or_tty->print_cr("Work queue overflow (benign) "
4842 "(pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT")",
4843 _par_pmc_remark_ovflw, _par_kac_ovflw);
4844 }
4845 _par_pmc_remark_ovflw = 0;
4846 _par_kac_ovflw = 0;
4847 }
4848 if (PrintCMSStatistics != 0) {
4849 if (_markStack._hit_limit > 0) {
4850 gclog_or_tty->print_cr(" (benign) Hit max stack size limit ("SIZE_FORMAT")",
4851 _markStack._hit_limit);
4852 }
4853 if (_markStack._failed_double > 0) {
4854 gclog_or_tty->print_cr(" (benign) Failed stack doubling ("SIZE_FORMAT"),"
4855 " current capacity "SIZE_FORMAT,
4856 _markStack._failed_double,
4857 _markStack.capacity());
4858 }
4859 }
4860 _markStack._hit_limit = 0;
4861 _markStack._failed_double = 0;
4863 if ((VerifyAfterGC || VerifyDuringGC) &&
4864 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4865 verify_after_remark();
4866 }
4868 // Change under the freelistLocks.
4869 _collectorState = Sweeping;
4870 // Call isAllClear() under bitMapLock
4871 assert(_modUnionTable.isAllClear(), "Should be clear by end of the"
4872 " final marking");
4873 if (UseAdaptiveSizePolicy) {
4874 size_policy()->checkpoint_roots_final_end(gch->gc_cause());
4875 }
4876 }
4878 // Parallel remark task
4879 class CMSParRemarkTask: public AbstractGangTask {
4880 CMSCollector* _collector;
4881 WorkGang* _workers;
4882 int _n_workers;
4883 CompactibleFreeListSpace* _cms_space;
4884 CompactibleFreeListSpace* _perm_space;
4886 // The per-thread work queues, available here for stealing.
4887 OopTaskQueueSet* _task_queues;
4888 ParallelTaskTerminator _term;
4890 public:
4891 CMSParRemarkTask(CMSCollector* collector,
4892 CompactibleFreeListSpace* cms_space,
4893 CompactibleFreeListSpace* perm_space,
4894 int n_workers, WorkGang* workers,
4895 OopTaskQueueSet* task_queues):
4896 AbstractGangTask("Rescan roots and grey objects in parallel"),
4897 _collector(collector),
4898 _cms_space(cms_space), _perm_space(perm_space),
4899 _n_workers(n_workers),
4900 _workers(workers),
4901 _task_queues(task_queues),
4902 _term(workers->total_workers(), task_queues) { }
4904 OopTaskQueueSet* task_queues() { return _task_queues; }
4906 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
4908 ParallelTaskTerminator* terminator() { return &_term; }
4910 void work(int i);
4912 private:
4913 // Work method in support of parallel rescan ... of young gen spaces
4914 void do_young_space_rescan(int i, Par_MarkRefsIntoAndScanClosure* cl,
4915 ContiguousSpace* space,
4916 HeapWord** chunk_array, size_t chunk_top);
4918 // ... of dirty cards in old space
4919 void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i,
4920 Par_MarkRefsIntoAndScanClosure* cl);
4922 // ... work stealing for the above
4923 void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed);
4924 };
4926 void CMSParRemarkTask::work(int i) {
4927 elapsedTimer _timer;
4928 ResourceMark rm;
4929 HandleMark hm;
4931 // ---------- rescan from roots --------------
4932 _timer.start();
4933 GenCollectedHeap* gch = GenCollectedHeap::heap();
4934 Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
4935 _collector->_span, _collector->ref_processor(),
4936 &(_collector->_markBitMap),
4937 work_queue(i), &(_collector->_revisitStack));
4939 // Rescan young gen roots first since these are likely
4940 // coarsely partitioned and may, on that account, constitute
4941 // the critical path; thus, it's best to start off that
4942 // work first.
4943 // ---------- young gen roots --------------
4944 {
4945 DefNewGeneration* dng = _collector->_young_gen->as_DefNewGeneration();
4946 EdenSpace* eden_space = dng->eden();
4947 ContiguousSpace* from_space = dng->from();
4948 ContiguousSpace* to_space = dng->to();
4950 HeapWord** eca = _collector->_eden_chunk_array;
4951 size_t ect = _collector->_eden_chunk_index;
4952 HeapWord** sca = _collector->_survivor_chunk_array;
4953 size_t sct = _collector->_survivor_chunk_index;
4955 assert(ect <= _collector->_eden_chunk_capacity, "out of bounds");
4956 assert(sct <= _collector->_survivor_chunk_capacity, "out of bounds");
4958 do_young_space_rescan(i, &par_mrias_cl, to_space, NULL, 0);
4959 do_young_space_rescan(i, &par_mrias_cl, from_space, sca, sct);
4960 do_young_space_rescan(i, &par_mrias_cl, eden_space, eca, ect);
4962 _timer.stop();
4963 if (PrintCMSStatistics != 0) {
4964 gclog_or_tty->print_cr(
4965 "Finished young gen rescan work in %dth thread: %3.3f sec",
4966 i, _timer.seconds());
4967 }
4968 }
4970 // ---------- remaining roots --------------
4971 _timer.reset();
4972 _timer.start();
4973 gch->gen_process_strong_roots(_collector->_cmsGen->level(),
4974 false, // yg was scanned above
4975 true, // collecting perm gen
4976 SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4977 NULL, &par_mrias_cl);
4978 _timer.stop();
4979 if (PrintCMSStatistics != 0) {
4980 gclog_or_tty->print_cr(
4981 "Finished remaining root rescan work in %dth thread: %3.3f sec",
4982 i, _timer.seconds());
4983 }
4985 // ---------- rescan dirty cards ------------
4986 _timer.reset();
4987 _timer.start();
4989 // Do the rescan tasks for each of the two spaces
4990 // (cms_space and perm_space) in turn.
4991 do_dirty_card_rescan_tasks(_cms_space, i, &par_mrias_cl);
4992 do_dirty_card_rescan_tasks(_perm_space, i, &par_mrias_cl);
4993 _timer.stop();
4994 if (PrintCMSStatistics != 0) {
4995 gclog_or_tty->print_cr(
4996 "Finished dirty card rescan work in %dth thread: %3.3f sec",
4997 i, _timer.seconds());
4998 }
5000 // ---------- steal work from other threads ...
5001 // ---------- ... and drain overflow list.
5002 _timer.reset();
5003 _timer.start();
5004 do_work_steal(i, &par_mrias_cl, _collector->hash_seed(i));
5005 _timer.stop();
5006 if (PrintCMSStatistics != 0) {
5007 gclog_or_tty->print_cr(
5008 "Finished work stealing in %dth thread: %3.3f sec",
5009 i, _timer.seconds());
5010 }
5011 }
5013 void
5014 CMSParRemarkTask::do_young_space_rescan(int i,
5015 Par_MarkRefsIntoAndScanClosure* cl, ContiguousSpace* space,
5016 HeapWord** chunk_array, size_t chunk_top) {
5017 // Until all tasks completed:
5018 // . claim an unclaimed task
5019 // . compute region boundaries corresponding to task claimed
5020 // using chunk_array
5021 // . par_oop_iterate(cl) over that region
5023 ResourceMark rm;
5024 HandleMark hm;
5026 SequentialSubTasksDone* pst = space->par_seq_tasks();
5027 assert(pst->valid(), "Uninitialized use?");
5029 int nth_task = 0;
5030 int n_tasks = pst->n_tasks();
5032 HeapWord *start, *end;
5033 while (!pst->is_task_claimed(/* reference */ nth_task)) {
5034 // We claimed task # nth_task; compute its boundaries.
5035 if (chunk_top == 0) { // no samples were taken
5036 assert(nth_task == 0 && n_tasks == 1, "Can have only 1 EdenSpace task");
5037 start = space->bottom();
5038 end = space->top();
5039 } else if (nth_task == 0) {
5040 start = space->bottom();
5041 end = chunk_array[nth_task];
5042 } else if (nth_task < (jint)chunk_top) {
5043 assert(nth_task >= 1, "Control point invariant");
5044 start = chunk_array[nth_task - 1];
5045 end = chunk_array[nth_task];
5046 } else {
5047 assert(nth_task == (jint)chunk_top, "Control point invariant");
5048 start = chunk_array[chunk_top - 1];
5049 end = space->top();
5050 }
5051 MemRegion mr(start, end);
5052 // Verify that mr is in space
5053 assert(mr.is_empty() || space->used_region().contains(mr),
5054 "Should be in space");
5055 // Verify that "start" is an object boundary
5056 assert(mr.is_empty() || oop(mr.start())->is_oop(),
5057 "Should be an oop");
5058 space->par_oop_iterate(mr, cl);
5059 }
5060 pst->all_tasks_completed();
5061 }
5063 void
5064 CMSParRemarkTask::do_dirty_card_rescan_tasks(
5065 CompactibleFreeListSpace* sp, int i,
5066 Par_MarkRefsIntoAndScanClosure* cl) {
5067 // Until all tasks completed:
5068 // . claim an unclaimed task
5069 // . compute region boundaries corresponding to task claimed
5070 // . transfer dirty bits ct->mut for that region
5071 // . apply rescanclosure to dirty mut bits for that region
5073 ResourceMark rm;
5074 HandleMark hm;
5076 OopTaskQueue* work_q = work_queue(i);
5077 ModUnionClosure modUnionClosure(&(_collector->_modUnionTable));
5078 // CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION!
5079 // CAUTION: This closure has state that persists across calls to
5080 // the work method dirty_range_iterate_clear() in that it has
5081 // imbedded in it a (subtype of) UpwardsObjectClosure. The
5082 // use of that state in the imbedded UpwardsObjectClosure instance
5083 // assumes that the cards are always iterated (even if in parallel
5084 // by several threads) in monotonically increasing order per each
5085 // thread. This is true of the implementation below which picks
5086 // card ranges (chunks) in monotonically increasing order globally
5087 // and, a-fortiori, in monotonically increasing order per thread
5088 // (the latter order being a subsequence of the former).
5089 // If the work code below is ever reorganized into a more chaotic
5090 // work-partitioning form than the current "sequential tasks"
5091 // paradigm, the use of that persistent state will have to be
5092 // revisited and modified appropriately. See also related
5093 // bug 4756801 work on which should examine this code to make
5094 // sure that the changes there do not run counter to the
5095 // assumptions made here and necessary for correctness and
5096 // efficiency. Note also that this code might yield inefficient
5097 // behaviour in the case of very large objects that span one or
5098 // more work chunks. Such objects would potentially be scanned
5099 // several times redundantly. Work on 4756801 should try and
5100 // address that performance anomaly if at all possible. XXX
5101 MemRegion full_span = _collector->_span;
5102 CMSBitMap* bm = &(_collector->_markBitMap); // shared
5103 CMSMarkStack* rs = &(_collector->_revisitStack); // shared
5104 MarkFromDirtyCardsClosure
5105 greyRescanClosure(_collector, full_span, // entire span of interest
5106 sp, bm, work_q, rs, cl);
5108 SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
5109 assert(pst->valid(), "Uninitialized use?");
5110 int nth_task = 0;
5111 const int alignment = CardTableModRefBS::card_size * BitsPerWord;
5112 MemRegion span = sp->used_region();
5113 HeapWord* start_addr = span.start();
5114 HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(),
5115 alignment);
5116 const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units
5117 assert((HeapWord*)round_to((intptr_t)start_addr, alignment) ==
5118 start_addr, "Check alignment");
5119 assert((size_t)round_to((intptr_t)chunk_size, alignment) ==
5120 chunk_size, "Check alignment");
5122 while (!pst->is_task_claimed(/* reference */ nth_task)) {
5123 // Having claimed the nth_task, compute corresponding mem-region,
5124 // which is a-fortiori aligned correctly (i.e. at a MUT bopundary).
5125 // The alignment restriction ensures that we do not need any
5126 // synchronization with other gang-workers while setting or
5127 // clearing bits in thus chunk of the MUT.
5128 MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size,
5129 start_addr + (nth_task+1)*chunk_size);
5130 // The last chunk's end might be way beyond end of the
5131 // used region. In that case pull back appropriately.
5132 if (this_span.end() > end_addr) {
5133 this_span.set_end(end_addr);
5134 assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)");
5135 }
5136 // Iterate over the dirty cards covering this chunk, marking them
5137 // precleaned, and setting the corresponding bits in the mod union
5138 // table. Since we have been careful to partition at Card and MUT-word
5139 // boundaries no synchronization is needed between parallel threads.
5140 _collector->_ct->ct_bs()->dirty_card_iterate(this_span,
5141 &modUnionClosure);
5143 // Having transferred these marks into the modUnionTable,
5144 // rescan the marked objects on the dirty cards in the modUnionTable.
5145 // Even if this is at a synchronous collection, the initial marking
5146 // may have been done during an asynchronous collection so there
5147 // may be dirty bits in the mod-union table.
5148 _collector->_modUnionTable.dirty_range_iterate_clear(
5149 this_span, &greyRescanClosure);
5150 _collector->_modUnionTable.verifyNoOneBitsInRange(
5151 this_span.start(),
5152 this_span.end());
5153 }
5154 pst->all_tasks_completed(); // declare that i am done
5155 }
5157 // . see if we can share work_queues with ParNew? XXX
5158 void
5159 CMSParRemarkTask::do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl,
5160 int* seed) {
5161 OopTaskQueue* work_q = work_queue(i);
5162 NOT_PRODUCT(int num_steals = 0;)
5163 oop obj_to_scan;
5164 CMSBitMap* bm = &(_collector->_markBitMap);
5165 size_t num_from_overflow_list =
5166 MIN2((size_t)work_q->max_elems()/4,
5167 (size_t)ParGCDesiredObjsFromOverflowList);
5169 while (true) {
5170 // Completely finish any left over work from (an) earlier round(s)
5171 cl->trim_queue(0);
5172 // Now check if there's any work in the overflow list
5173 if (_collector->par_take_from_overflow_list(num_from_overflow_list,
5174 work_q)) {
5175 // found something in global overflow list;
5176 // not yet ready to go stealing work from others.
5177 // We'd like to assert(work_q->size() != 0, ...)
5178 // because we just took work from the overflow list,
5179 // but of course we can't since all of that could have
5180 // been already stolen from us.
5181 // "He giveth and He taketh away."
5182 continue;
5183 }
5184 // Verify that we have no work before we resort to stealing
5185 assert(work_q->size() == 0, "Have work, shouldn't steal");
5186 // Try to steal from other queues that have work
5187 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5188 NOT_PRODUCT(num_steals++;)
5189 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5190 assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5191 // Do scanning work
5192 obj_to_scan->oop_iterate(cl);
5193 // Loop around, finish this work, and try to steal some more
5194 } else if (terminator()->offer_termination()) {
5195 break; // nirvana from the infinite cycle
5196 }
5197 }
5198 NOT_PRODUCT(
5199 if (PrintCMSStatistics != 0) {
5200 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
5201 }
5202 )
5203 assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
5204 "Else our work is not yet done");
5205 }
5207 // Return a thread-local PLAB recording array, as appropriate.
5208 void* CMSCollector::get_data_recorder(int thr_num) {
5209 if (_survivor_plab_array != NULL &&
5210 (CMSPLABRecordAlways ||
5211 (_collectorState > Marking && _collectorState < FinalMarking))) {
5212 assert(thr_num < (int)ParallelGCThreads, "thr_num is out of bounds");
5213 ChunkArray* ca = &_survivor_plab_array[thr_num];
5214 ca->reset(); // clear it so that fresh data is recorded
5215 return (void*) ca;
5216 } else {
5217 return NULL;
5218 }
5219 }
5221 // Reset all the thread-local PLAB recording arrays
5222 void CMSCollector::reset_survivor_plab_arrays() {
5223 for (uint i = 0; i < ParallelGCThreads; i++) {
5224 _survivor_plab_array[i].reset();
5225 }
5226 }
5228 // Merge the per-thread plab arrays into the global survivor chunk
5229 // array which will provide the partitioning of the survivor space
5230 // for CMS rescan.
5231 void CMSCollector::merge_survivor_plab_arrays(ContiguousSpace* surv) {
5232 assert(_survivor_plab_array != NULL, "Error");
5233 assert(_survivor_chunk_array != NULL, "Error");
5234 assert(_collectorState == FinalMarking, "Error");
5235 for (uint j = 0; j < ParallelGCThreads; j++) {
5236 _cursor[j] = 0;
5237 }
5238 HeapWord* top = surv->top();
5239 size_t i;
5240 for (i = 0; i < _survivor_chunk_capacity; i++) { // all sca entries
5241 HeapWord* min_val = top; // Higher than any PLAB address
5242 uint min_tid = 0; // position of min_val this round
5243 for (uint j = 0; j < ParallelGCThreads; j++) {
5244 ChunkArray* cur_sca = &_survivor_plab_array[j];
5245 if (_cursor[j] == cur_sca->end()) {
5246 continue;
5247 }
5248 assert(_cursor[j] < cur_sca->end(), "ctl pt invariant");
5249 HeapWord* cur_val = cur_sca->nth(_cursor[j]);
5250 assert(surv->used_region().contains(cur_val), "Out of bounds value");
5251 if (cur_val < min_val) {
5252 min_tid = j;
5253 min_val = cur_val;
5254 } else {
5255 assert(cur_val < top, "All recorded addresses should be less");
5256 }
5257 }
5258 // At this point min_val and min_tid are respectively
5259 // the least address in _survivor_plab_array[j]->nth(_cursor[j])
5260 // and the thread (j) that witnesses that address.
5261 // We record this address in the _survivor_chunk_array[i]
5262 // and increment _cursor[min_tid] prior to the next round i.
5263 if (min_val == top) {
5264 break;
5265 }
5266 _survivor_chunk_array[i] = min_val;
5267 _cursor[min_tid]++;
5268 }
5269 // We are all done; record the size of the _survivor_chunk_array
5270 _survivor_chunk_index = i; // exclusive: [0, i)
5271 if (PrintCMSStatistics > 0) {
5272 gclog_or_tty->print(" (Survivor:" SIZE_FORMAT "chunks) ", i);
5273 }
5274 // Verify that we used up all the recorded entries
5275 #ifdef ASSERT
5276 size_t total = 0;
5277 for (uint j = 0; j < ParallelGCThreads; j++) {
5278 assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
5279 total += _cursor[j];
5280 }
5281 assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
5282 // Check that the merged array is in sorted order
5283 if (total > 0) {
5284 for (size_t i = 0; i < total - 1; i++) {
5285 if (PrintCMSStatistics > 0) {
5286 gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
5287 i, _survivor_chunk_array[i]);
5288 }
5289 assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
5290 "Not sorted");
5291 }
5292 }
5293 #endif // ASSERT
5294 }
5296 // Set up the space's par_seq_tasks structure for work claiming
5297 // for parallel rescan of young gen.
5298 // See ParRescanTask where this is currently used.
5299 void
5300 CMSCollector::
5301 initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
5302 assert(n_threads > 0, "Unexpected n_threads argument");
5303 DefNewGeneration* dng = (DefNewGeneration*)_young_gen;
5305 // Eden space
5306 {
5307 SequentialSubTasksDone* pst = dng->eden()->par_seq_tasks();
5308 assert(!pst->valid(), "Clobbering existing data?");
5309 // Each valid entry in [0, _eden_chunk_index) represents a task.
5310 size_t n_tasks = _eden_chunk_index + 1;
5311 assert(n_tasks == 1 || _eden_chunk_array != NULL, "Error");
5312 pst->set_par_threads(n_threads);
5313 pst->set_n_tasks((int)n_tasks);
5314 }
5316 // Merge the survivor plab arrays into _survivor_chunk_array
5317 if (_survivor_plab_array != NULL) {
5318 merge_survivor_plab_arrays(dng->from());
5319 } else {
5320 assert(_survivor_chunk_index == 0, "Error");
5321 }
5323 // To space
5324 {
5325 SequentialSubTasksDone* pst = dng->to()->par_seq_tasks();
5326 assert(!pst->valid(), "Clobbering existing data?");
5327 pst->set_par_threads(n_threads);
5328 pst->set_n_tasks(1);
5329 assert(pst->valid(), "Error");
5330 }
5332 // From space
5333 {
5334 SequentialSubTasksDone* pst = dng->from()->par_seq_tasks();
5335 assert(!pst->valid(), "Clobbering existing data?");
5336 size_t n_tasks = _survivor_chunk_index + 1;
5337 assert(n_tasks == 1 || _survivor_chunk_array != NULL, "Error");
5338 pst->set_par_threads(n_threads);
5339 pst->set_n_tasks((int)n_tasks);
5340 assert(pst->valid(), "Error");
5341 }
5342 }
5344 // Parallel version of remark
5345 void CMSCollector::do_remark_parallel() {
5346 GenCollectedHeap* gch = GenCollectedHeap::heap();
5347 WorkGang* workers = gch->workers();
5348 assert(workers != NULL, "Need parallel worker threads.");
5349 int n_workers = workers->total_workers();
5350 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
5351 CompactibleFreeListSpace* perm_space = _permGen->cmsSpace();
5353 CMSParRemarkTask tsk(this,
5354 cms_space, perm_space,
5355 n_workers, workers, task_queues());
5357 // Set up for parallel process_strong_roots work.
5358 gch->set_par_threads(n_workers);
5359 gch->change_strong_roots_parity();
5360 // We won't be iterating over the cards in the card table updating
5361 // the younger_gen cards, so we shouldn't call the following else
5362 // the verification code as well as subsequent younger_refs_iterate
5363 // code would get confused. XXX
5364 // gch->rem_set()->prepare_for_younger_refs_iterate(true); // parallel
5366 // The young gen rescan work will not be done as part of
5367 // process_strong_roots (which currently doesn't knw how to
5368 // parallelize such a scan), but rather will be broken up into
5369 // a set of parallel tasks (via the sampling that the [abortable]
5370 // preclean phase did of EdenSpace, plus the [two] tasks of
5371 // scanning the [two] survivor spaces. Further fine-grain
5372 // parallelization of the scanning of the survivor spaces
5373 // themselves, and of precleaning of the younger gen itself
5374 // is deferred to the future.
5375 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
5377 // The dirty card rescan work is broken up into a "sequence"
5378 // of parallel tasks (per constituent space) that are dynamically
5379 // claimed by the parallel threads.
5380 cms_space->initialize_sequential_subtasks_for_rescan(n_workers);
5381 perm_space->initialize_sequential_subtasks_for_rescan(n_workers);
5383 // It turns out that even when we're using 1 thread, doing the work in a
5384 // separate thread causes wide variance in run times. We can't help this
5385 // in the multi-threaded case, but we special-case n=1 here to get
5386 // repeatable measurements of the 1-thread overhead of the parallel code.
5387 if (n_workers > 1) {
5388 // Make refs discovery MT-safe
5389 ReferenceProcessorMTMutator mt(ref_processor(), true);
5390 workers->run_task(&tsk);
5391 } else {
5392 tsk.work(0);
5393 }
5394 gch->set_par_threads(0); // 0 ==> non-parallel.
5395 // restore, single-threaded for now, any preserved marks
5396 // as a result of work_q overflow
5397 restore_preserved_marks_if_any();
5398 }
5400 // Non-parallel version of remark
5401 void CMSCollector::do_remark_non_parallel() {
5402 ResourceMark rm;
5403 HandleMark hm;
5404 GenCollectedHeap* gch = GenCollectedHeap::heap();
5405 MarkRefsIntoAndScanClosure
5406 mrias_cl(_span, ref_processor(), &_markBitMap, &_modUnionTable,
5407 &_markStack, &_revisitStack, this,
5408 false /* should_yield */, false /* not precleaning */);
5409 MarkFromDirtyCardsClosure
5410 markFromDirtyCardsClosure(this, _span,
5411 NULL, // space is set further below
5412 &_markBitMap, &_markStack, &_revisitStack,
5413 &mrias_cl);
5414 {
5415 TraceTime t("grey object rescan", PrintGCDetails, false, gclog_or_tty);
5416 // Iterate over the dirty cards, marking them precleaned, and
5417 // setting the corresponding bits in the mod union table.
5418 {
5419 ModUnionClosure modUnionClosure(&_modUnionTable);
5420 _ct->ct_bs()->dirty_card_iterate(
5421 _cmsGen->used_region(),
5422 &modUnionClosure);
5423 _ct->ct_bs()->dirty_card_iterate(
5424 _permGen->used_region(),
5425 &modUnionClosure);
5426 }
5427 // Having transferred these marks into the modUnionTable, we just need
5428 // to rescan the marked objects on the dirty cards in the modUnionTable.
5429 // The initial marking may have been done during an asynchronous
5430 // collection so there may be dirty bits in the mod-union table.
5431 const int alignment =
5432 CardTableModRefBS::card_size * BitsPerWord;
5433 {
5434 // ... First handle dirty cards in CMS gen
5435 markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
5436 MemRegion ur = _cmsGen->used_region();
5437 HeapWord* lb = ur.start();
5438 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
5439 MemRegion cms_span(lb, ub);
5440 _modUnionTable.dirty_range_iterate_clear(cms_span,
5441 &markFromDirtyCardsClosure);
5442 verify_work_stacks_empty();
5443 if (PrintCMSStatistics != 0) {
5444 gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ",
5445 markFromDirtyCardsClosure.num_dirty_cards());
5446 }
5447 }
5448 {
5449 // .. and then repeat for dirty cards in perm gen
5450 markFromDirtyCardsClosure.set_space(_permGen->cmsSpace());
5451 MemRegion ur = _permGen->used_region();
5452 HeapWord* lb = ur.start();
5453 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
5454 MemRegion perm_span(lb, ub);
5455 _modUnionTable.dirty_range_iterate_clear(perm_span,
5456 &markFromDirtyCardsClosure);
5457 verify_work_stacks_empty();
5458 if (PrintCMSStatistics != 0) {
5459 gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in perm gen) ",
5460 markFromDirtyCardsClosure.num_dirty_cards());
5461 }
5462 }
5463 }
5464 if (VerifyDuringGC &&
5465 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
5466 HandleMark hm; // Discard invalid handles created during verification
5467 Universe::verify(true);
5468 }
5469 {
5470 TraceTime t("root rescan", PrintGCDetails, false, gclog_or_tty);
5472 verify_work_stacks_empty();
5474 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
5475 gch->gen_process_strong_roots(_cmsGen->level(),
5476 true, // younger gens as roots
5477 true, // collecting perm gen
5478 SharedHeap::ScanningOption(roots_scanning_options()),
5479 NULL, &mrias_cl);
5480 }
5481 verify_work_stacks_empty();
5482 // Restore evacuated mark words, if any, used for overflow list links
5483 if (!CMSOverflowEarlyRestoration) {
5484 restore_preserved_marks_if_any();
5485 }
5486 verify_overflow_empty();
5487 }
5489 ////////////////////////////////////////////////////////
5490 // Parallel Reference Processing Task Proxy Class
5491 ////////////////////////////////////////////////////////
5492 class CMSRefProcTaskProxy: public AbstractGangTask {
5493 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
5494 CMSCollector* _collector;
5495 CMSBitMap* _mark_bit_map;
5496 MemRegion _span;
5497 OopTaskQueueSet* _task_queues;
5498 ParallelTaskTerminator _term;
5499 ProcessTask& _task;
5501 public:
5502 CMSRefProcTaskProxy(ProcessTask& task,
5503 CMSCollector* collector,
5504 const MemRegion& span,
5505 CMSBitMap* mark_bit_map,
5506 int total_workers,
5507 OopTaskQueueSet* task_queues):
5508 AbstractGangTask("Process referents by policy in parallel"),
5509 _task(task),
5510 _collector(collector), _span(span), _mark_bit_map(mark_bit_map),
5511 _task_queues(task_queues),
5512 _term(total_workers, task_queues)
5513 { }
5515 OopTaskQueueSet* task_queues() { return _task_queues; }
5517 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
5519 ParallelTaskTerminator* terminator() { return &_term; }
5521 void do_work_steal(int i,
5522 CMSParDrainMarkingStackClosure* drain,
5523 CMSParKeepAliveClosure* keep_alive,
5524 int* seed);
5526 virtual void work(int i);
5527 };
5529 void CMSRefProcTaskProxy::work(int i) {
5530 CMSParKeepAliveClosure par_keep_alive(_collector, _span,
5531 _mark_bit_map, work_queue(i));
5532 CMSParDrainMarkingStackClosure par_drain_stack(_collector, _span,
5533 _mark_bit_map, work_queue(i));
5534 CMSIsAliveClosure is_alive_closure(_mark_bit_map);
5535 _task.work(i, is_alive_closure, par_keep_alive, par_drain_stack);
5536 if (_task.marks_oops_alive()) {
5537 do_work_steal(i, &par_drain_stack, &par_keep_alive,
5538 _collector->hash_seed(i));
5539 }
5540 assert(work_queue(i)->size() == 0, "work_queue should be empty");
5541 assert(_collector->_overflow_list == NULL, "non-empty _overflow_list");
5542 }
5544 class CMSRefEnqueueTaskProxy: public AbstractGangTask {
5545 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
5546 EnqueueTask& _task;
5548 public:
5549 CMSRefEnqueueTaskProxy(EnqueueTask& task)
5550 : AbstractGangTask("Enqueue reference objects in parallel"),
5551 _task(task)
5552 { }
5554 virtual void work(int i)
5555 {
5556 _task.work(i);
5557 }
5558 };
5560 CMSParKeepAliveClosure::CMSParKeepAliveClosure(CMSCollector* collector,
5561 MemRegion span, CMSBitMap* bit_map, OopTaskQueue* work_queue):
5562 _collector(collector),
5563 _span(span),
5564 _bit_map(bit_map),
5565 _work_queue(work_queue),
5566 _mark_and_push(collector, span, bit_map, work_queue),
5567 _low_water_mark(MIN2((uint)(work_queue->max_elems()/4),
5568 (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads)))
5569 { }
5571 // . see if we can share work_queues with ParNew? XXX
5572 void CMSRefProcTaskProxy::do_work_steal(int i,
5573 CMSParDrainMarkingStackClosure* drain,
5574 CMSParKeepAliveClosure* keep_alive,
5575 int* seed) {
5576 OopTaskQueue* work_q = work_queue(i);
5577 NOT_PRODUCT(int num_steals = 0;)
5578 oop obj_to_scan;
5579 size_t num_from_overflow_list =
5580 MIN2((size_t)work_q->max_elems()/4,
5581 (size_t)ParGCDesiredObjsFromOverflowList);
5583 while (true) {
5584 // Completely finish any left over work from (an) earlier round(s)
5585 drain->trim_queue(0);
5586 // Now check if there's any work in the overflow list
5587 if (_collector->par_take_from_overflow_list(num_from_overflow_list,
5588 work_q)) {
5589 // Found something in global overflow list;
5590 // not yet ready to go stealing work from others.
5591 // We'd like to assert(work_q->size() != 0, ...)
5592 // because we just took work from the overflow list,
5593 // but of course we can't, since all of that might have
5594 // been already stolen from us.
5595 continue;
5596 }
5597 // Verify that we have no work before we resort to stealing
5598 assert(work_q->size() == 0, "Have work, shouldn't steal");
5599 // Try to steal from other queues that have work
5600 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5601 NOT_PRODUCT(num_steals++;)
5602 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5603 assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5604 // Do scanning work
5605 obj_to_scan->oop_iterate(keep_alive);
5606 // Loop around, finish this work, and try to steal some more
5607 } else if (terminator()->offer_termination()) {
5608 break; // nirvana from the infinite cycle
5609 }
5610 }
5611 NOT_PRODUCT(
5612 if (PrintCMSStatistics != 0) {
5613 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
5614 }
5615 )
5616 }
5618 void CMSRefProcTaskExecutor::execute(ProcessTask& task)
5619 {
5620 GenCollectedHeap* gch = GenCollectedHeap::heap();
5621 WorkGang* workers = gch->workers();
5622 assert(workers != NULL, "Need parallel worker threads.");
5623 int n_workers = workers->total_workers();
5624 CMSRefProcTaskProxy rp_task(task, &_collector,
5625 _collector.ref_processor()->span(),
5626 _collector.markBitMap(),
5627 n_workers, _collector.task_queues());
5628 workers->run_task(&rp_task);
5629 }
5631 void CMSRefProcTaskExecutor::execute(EnqueueTask& task)
5632 {
5634 GenCollectedHeap* gch = GenCollectedHeap::heap();
5635 WorkGang* workers = gch->workers();
5636 assert(workers != NULL, "Need parallel worker threads.");
5637 CMSRefEnqueueTaskProxy enq_task(task);
5638 workers->run_task(&enq_task);
5639 }
5641 void CMSCollector::refProcessingWork(bool asynch, bool clear_all_soft_refs) {
5643 ResourceMark rm;
5644 HandleMark hm;
5645 ReferencePolicy* soft_ref_policy;
5647 assert(!ref_processor()->enqueuing_is_done(), "Enqueuing should not be complete");
5648 // Process weak references.
5649 if (clear_all_soft_refs) {
5650 soft_ref_policy = new AlwaysClearPolicy();
5651 } else {
5652 #ifdef COMPILER2
5653 soft_ref_policy = new LRUMaxHeapPolicy();
5654 #else
5655 soft_ref_policy = new LRUCurrentHeapPolicy();
5656 #endif // COMPILER2
5657 }
5658 verify_work_stacks_empty();
5660 ReferenceProcessor* rp = ref_processor();
5661 assert(rp->span().equals(_span), "Spans should be equal");
5662 CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap,
5663 &_markStack);
5664 CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this,
5665 _span, &_markBitMap, &_markStack,
5666 &cmsKeepAliveClosure);
5667 {
5668 TraceTime t("weak refs processing", PrintGCDetails, false, gclog_or_tty);
5669 if (rp->processing_is_mt()) {
5670 CMSRefProcTaskExecutor task_executor(*this);
5671 rp->process_discovered_references(soft_ref_policy,
5672 &_is_alive_closure,
5673 &cmsKeepAliveClosure,
5674 &cmsDrainMarkingStackClosure,
5675 &task_executor);
5676 } else {
5677 rp->process_discovered_references(soft_ref_policy,
5678 &_is_alive_closure,
5679 &cmsKeepAliveClosure,
5680 &cmsDrainMarkingStackClosure,
5681 NULL);
5682 }
5683 verify_work_stacks_empty();
5684 }
5686 if (should_unload_classes()) {
5687 {
5688 TraceTime t("class unloading", PrintGCDetails, false, gclog_or_tty);
5690 // Follow SystemDictionary roots and unload classes
5691 bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure);
5693 // Follow CodeCache roots and unload any methods marked for unloading
5694 CodeCache::do_unloading(&_is_alive_closure,
5695 &cmsKeepAliveClosure,
5696 purged_class);
5698 cmsDrainMarkingStackClosure.do_void();
5699 verify_work_stacks_empty();
5701 // Update subklass/sibling/implementor links in KlassKlass descendants
5702 assert(!_revisitStack.isEmpty(), "revisit stack should not be empty");
5703 oop k;
5704 while ((k = _revisitStack.pop()) != NULL) {
5705 ((Klass*)(oopDesc*)k)->follow_weak_klass_links(
5706 &_is_alive_closure,
5707 &cmsKeepAliveClosure);
5708 }
5709 assert(!ClassUnloading ||
5710 (_markStack.isEmpty() && overflow_list_is_empty()),
5711 "Should not have found new reachable objects");
5712 assert(_revisitStack.isEmpty(), "revisit stack should have been drained");
5713 cmsDrainMarkingStackClosure.do_void();
5714 verify_work_stacks_empty();
5715 }
5717 {
5718 TraceTime t("scrub symbol & string tables", PrintGCDetails, false, gclog_or_tty);
5719 // Now clean up stale oops in SymbolTable and StringTable
5720 SymbolTable::unlink(&_is_alive_closure);
5721 StringTable::unlink(&_is_alive_closure);
5722 }
5723 }
5725 verify_work_stacks_empty();
5726 // Restore any preserved marks as a result of mark stack or
5727 // work queue overflow
5728 restore_preserved_marks_if_any(); // done single-threaded for now
5730 rp->set_enqueuing_is_done(true);
5731 if (rp->processing_is_mt()) {
5732 CMSRefProcTaskExecutor task_executor(*this);
5733 rp->enqueue_discovered_references(&task_executor);
5734 } else {
5735 rp->enqueue_discovered_references(NULL);
5736 }
5737 rp->verify_no_references_recorded();
5738 assert(!rp->discovery_enabled(), "should have been disabled");
5740 // JVMTI object tagging is based on JNI weak refs. If any of these
5741 // refs were cleared then JVMTI needs to update its maps and
5742 // maybe post ObjectFrees to agents.
5743 JvmtiExport::cms_ref_processing_epilogue();
5744 }
5746 #ifndef PRODUCT
5747 void CMSCollector::check_correct_thread_executing() {
5748 Thread* t = Thread::current();
5749 // Only the VM thread or the CMS thread should be here.
5750 assert(t->is_ConcurrentGC_thread() || t->is_VM_thread(),
5751 "Unexpected thread type");
5752 // If this is the vm thread, the foreground process
5753 // should not be waiting. Note that _foregroundGCIsActive is
5754 // true while the foreground collector is waiting.
5755 if (_foregroundGCShouldWait) {
5756 // We cannot be the VM thread
5757 assert(t->is_ConcurrentGC_thread(),
5758 "Should be CMS thread");
5759 } else {
5760 // We can be the CMS thread only if we are in a stop-world
5761 // phase of CMS collection.
5762 if (t->is_ConcurrentGC_thread()) {
5763 assert(_collectorState == InitialMarking ||
5764 _collectorState == FinalMarking,
5765 "Should be a stop-world phase");
5766 // The CMS thread should be holding the CMS_token.
5767 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
5768 "Potential interference with concurrently "
5769 "executing VM thread");
5770 }
5771 }
5772 }
5773 #endif
5775 void CMSCollector::sweep(bool asynch) {
5776 assert(_collectorState == Sweeping, "just checking");
5777 check_correct_thread_executing();
5778 verify_work_stacks_empty();
5779 verify_overflow_empty();
5780 incrementSweepCount();
5781 _sweep_timer.stop();
5782 _sweep_estimate.sample(_sweep_timer.seconds());
5783 size_policy()->avg_cms_free_at_sweep()->sample(_cmsGen->free());
5785 // PermGen verification support: If perm gen sweeping is disabled in
5786 // this cycle, we preserve the perm gen object "deadness" information
5787 // in the perm_gen_verify_bit_map. In order to do that we traverse
5788 // all blocks in perm gen and mark all dead objects.
5789 if (verifying() && !should_unload_classes()) {
5790 assert(perm_gen_verify_bit_map()->sizeInBits() != 0,
5791 "Should have already been allocated");
5792 MarkDeadObjectsClosure mdo(this, _permGen->cmsSpace(),
5793 markBitMap(), perm_gen_verify_bit_map());
5794 if (asynch) {
5795 CMSTokenSyncWithLocks ts(true, _permGen->freelistLock(),
5796 bitMapLock());
5797 _permGen->cmsSpace()->blk_iterate(&mdo);
5798 } else {
5799 // In the case of synchronous sweep, we already have
5800 // the requisite locks/tokens.
5801 _permGen->cmsSpace()->blk_iterate(&mdo);
5802 }
5803 }
5805 if (asynch) {
5806 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
5807 CMSPhaseAccounting pa(this, "sweep", !PrintGCDetails);
5808 // First sweep the old gen then the perm gen
5809 {
5810 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
5811 bitMapLock());
5812 sweepWork(_cmsGen, asynch);
5813 }
5815 // Now repeat for perm gen
5816 if (should_unload_classes()) {
5817 CMSTokenSyncWithLocks ts(true, _permGen->freelistLock(),
5818 bitMapLock());
5819 sweepWork(_permGen, asynch);
5820 }
5822 // Update Universe::_heap_*_at_gc figures.
5823 // We need all the free list locks to make the abstract state
5824 // transition from Sweeping to Resetting. See detailed note
5825 // further below.
5826 {
5827 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
5828 _permGen->freelistLock());
5829 // Update heap occupancy information which is used as
5830 // input to soft ref clearing policy at the next gc.
5831 Universe::update_heap_info_at_gc();
5832 _collectorState = Resizing;
5833 }
5834 } else {
5835 // already have needed locks
5836 sweepWork(_cmsGen, asynch);
5838 if (should_unload_classes()) {
5839 sweepWork(_permGen, asynch);
5840 }
5841 // Update heap occupancy information which is used as
5842 // input to soft ref clearing policy at the next gc.
5843 Universe::update_heap_info_at_gc();
5844 _collectorState = Resizing;
5845 }
5846 verify_work_stacks_empty();
5847 verify_overflow_empty();
5849 _sweep_timer.reset();
5850 _sweep_timer.start();
5852 update_time_of_last_gc(os::javaTimeMillis());
5854 // NOTE on abstract state transitions:
5855 // Mutators allocate-live and/or mark the mod-union table dirty
5856 // based on the state of the collection. The former is done in
5857 // the interval [Marking, Sweeping] and the latter in the interval
5858 // [Marking, Sweeping). Thus the transitions into the Marking state
5859 // and out of the Sweeping state must be synchronously visible
5860 // globally to the mutators.
5861 // The transition into the Marking state happens with the world
5862 // stopped so the mutators will globally see it. Sweeping is
5863 // done asynchronously by the background collector so the transition
5864 // from the Sweeping state to the Resizing state must be done
5865 // under the freelistLock (as is the check for whether to
5866 // allocate-live and whether to dirty the mod-union table).
5867 assert(_collectorState == Resizing, "Change of collector state to"
5868 " Resizing must be done under the freelistLocks (plural)");
5870 // Now that sweeping has been completed, if the GCH's
5871 // incremental_collection_will_fail flag is set, clear it,
5872 // thus inviting a younger gen collection to promote into
5873 // this generation. If such a promotion may still fail,
5874 // the flag will be set again when a young collection is
5875 // attempted.
5876 // I think the incremental_collection_will_fail flag's use
5877 // is specific to a 2 generation collection policy, so i'll
5878 // assert that that's the configuration we are operating within.
5879 // The use of the flag can and should be generalized appropriately
5880 // in the future to deal with a general n-generation system.
5882 GenCollectedHeap* gch = GenCollectedHeap::heap();
5883 assert(gch->collector_policy()->is_two_generation_policy(),
5884 "Resetting of incremental_collection_will_fail flag"
5885 " may be incorrect otherwise");
5886 gch->clear_incremental_collection_will_fail();
5887 gch->update_full_collections_completed(_collection_count_start);
5888 }
5890 // FIX ME!!! Looks like this belongs in CFLSpace, with
5891 // CMSGen merely delegating to it.
5892 void ConcurrentMarkSweepGeneration::setNearLargestChunk() {
5893 double nearLargestPercent = 0.999;
5894 HeapWord* minAddr = _cmsSpace->bottom();
5895 HeapWord* largestAddr =
5896 (HeapWord*) _cmsSpace->dictionary()->findLargestDict();
5897 if (largestAddr == 0) {
5898 // The dictionary appears to be empty. In this case
5899 // try to coalesce at the end of the heap.
5900 largestAddr = _cmsSpace->end();
5901 }
5902 size_t largestOffset = pointer_delta(largestAddr, minAddr);
5903 size_t nearLargestOffset =
5904 (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
5905 _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
5906 }
5908 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
5909 return addr >= _cmsSpace->nearLargestChunk();
5910 }
5912 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
5913 return _cmsSpace->find_chunk_at_end();
5914 }
5916 void ConcurrentMarkSweepGeneration::update_gc_stats(int current_level,
5917 bool full) {
5918 // The next lower level has been collected. Gather any statistics
5919 // that are of interest at this point.
5920 if (!full && (current_level + 1) == level()) {
5921 // Gather statistics on the young generation collection.
5922 collector()->stats().record_gc0_end(used());
5923 }
5924 }
5926 CMSAdaptiveSizePolicy* ConcurrentMarkSweepGeneration::size_policy() {
5927 GenCollectedHeap* gch = GenCollectedHeap::heap();
5928 assert(gch->kind() == CollectedHeap::GenCollectedHeap,
5929 "Wrong type of heap");
5930 CMSAdaptiveSizePolicy* sp = (CMSAdaptiveSizePolicy*)
5931 gch->gen_policy()->size_policy();
5932 assert(sp->is_gc_cms_adaptive_size_policy(),
5933 "Wrong type of size policy");
5934 return sp;
5935 }
5937 void ConcurrentMarkSweepGeneration::rotate_debug_collection_type() {
5938 if (PrintGCDetails && Verbose) {
5939 gclog_or_tty->print("Rotate from %d ", _debug_collection_type);
5940 }
5941 _debug_collection_type = (CollectionTypes) (_debug_collection_type + 1);
5942 _debug_collection_type =
5943 (CollectionTypes) (_debug_collection_type % Unknown_collection_type);
5944 if (PrintGCDetails && Verbose) {
5945 gclog_or_tty->print_cr("to %d ", _debug_collection_type);
5946 }
5947 }
5949 void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* gen,
5950 bool asynch) {
5951 // We iterate over the space(s) underlying this generation,
5952 // checking the mark bit map to see if the bits corresponding
5953 // to specific blocks are marked or not. Blocks that are
5954 // marked are live and are not swept up. All remaining blocks
5955 // are swept up, with coalescing on-the-fly as we sweep up
5956 // contiguous free and/or garbage blocks:
5957 // We need to ensure that the sweeper synchronizes with allocators
5958 // and stop-the-world collectors. In particular, the following
5959 // locks are used:
5960 // . CMS token: if this is held, a stop the world collection cannot occur
5961 // . freelistLock: if this is held no allocation can occur from this
5962 // generation by another thread
5963 // . bitMapLock: if this is held, no other thread can access or update
5964 //
5966 // Note that we need to hold the freelistLock if we use
5967 // block iterate below; else the iterator might go awry if
5968 // a mutator (or promotion) causes block contents to change
5969 // (for instance if the allocator divvies up a block).
5970 // If we hold the free list lock, for all practical purposes
5971 // young generation GC's can't occur (they'll usually need to
5972 // promote), so we might as well prevent all young generation
5973 // GC's while we do a sweeping step. For the same reason, we might
5974 // as well take the bit map lock for the entire duration
5976 // check that we hold the requisite locks
5977 assert(have_cms_token(), "Should hold cms token");
5978 assert( (asynch && ConcurrentMarkSweepThread::cms_thread_has_cms_token())
5979 || (!asynch && ConcurrentMarkSweepThread::vm_thread_has_cms_token()),
5980 "Should possess CMS token to sweep");
5981 assert_lock_strong(gen->freelistLock());
5982 assert_lock_strong(bitMapLock());
5984 assert(!_sweep_timer.is_active(), "Was switched off in an outer context");
5985 gen->cmsSpace()->beginSweepFLCensus((float)(_sweep_timer.seconds()),
5986 _sweep_estimate.padded_average());
5987 gen->setNearLargestChunk();
5989 {
5990 SweepClosure sweepClosure(this, gen, &_markBitMap,
5991 CMSYield && asynch);
5992 gen->cmsSpace()->blk_iterate_careful(&sweepClosure);
5993 // We need to free-up/coalesce garbage/blocks from a
5994 // co-terminal free run. This is done in the SweepClosure
5995 // destructor; so, do not remove this scope, else the
5996 // end-of-sweep-census below will be off by a little bit.
5997 }
5998 gen->cmsSpace()->sweep_completed();
5999 gen->cmsSpace()->endSweepFLCensus(sweepCount());
6000 if (should_unload_classes()) { // unloaded classes this cycle,
6001 _concurrent_cycles_since_last_unload = 0; // ... reset count
6002 } else { // did not unload classes,
6003 _concurrent_cycles_since_last_unload++; // ... increment count
6004 }
6005 }
6007 // Reset CMS data structures (for now just the marking bit map)
6008 // preparatory for the next cycle.
6009 void CMSCollector::reset(bool asynch) {
6010 GenCollectedHeap* gch = GenCollectedHeap::heap();
6011 CMSAdaptiveSizePolicy* sp = size_policy();
6012 AdaptiveSizePolicyOutput(sp, gch->total_collections());
6013 if (asynch) {
6014 CMSTokenSyncWithLocks ts(true, bitMapLock());
6016 // If the state is not "Resetting", the foreground thread
6017 // has done a collection and the resetting.
6018 if (_collectorState != Resetting) {
6019 assert(_collectorState == Idling, "The state should only change"
6020 " because the foreground collector has finished the collection");
6021 return;
6022 }
6024 // Clear the mark bitmap (no grey objects to start with)
6025 // for the next cycle.
6026 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
6027 CMSPhaseAccounting cmspa(this, "reset", !PrintGCDetails);
6029 HeapWord* curAddr = _markBitMap.startWord();
6030 while (curAddr < _markBitMap.endWord()) {
6031 size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr);
6032 MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining));
6033 _markBitMap.clear_large_range(chunk);
6034 if (ConcurrentMarkSweepThread::should_yield() &&
6035 !foregroundGCIsActive() &&
6036 CMSYield) {
6037 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6038 "CMS thread should hold CMS token");
6039 assert_lock_strong(bitMapLock());
6040 bitMapLock()->unlock();
6041 ConcurrentMarkSweepThread::desynchronize(true);
6042 ConcurrentMarkSweepThread::acknowledge_yield_request();
6043 stopTimer();
6044 if (PrintCMSStatistics != 0) {
6045 incrementYields();
6046 }
6047 icms_wait();
6049 // See the comment in coordinator_yield()
6050 for (unsigned i = 0; i < CMSYieldSleepCount &&
6051 ConcurrentMarkSweepThread::should_yield() &&
6052 !CMSCollector::foregroundGCIsActive(); ++i) {
6053 os::sleep(Thread::current(), 1, false);
6054 ConcurrentMarkSweepThread::acknowledge_yield_request();
6055 }
6057 ConcurrentMarkSweepThread::synchronize(true);
6058 bitMapLock()->lock_without_safepoint_check();
6059 startTimer();
6060 }
6061 curAddr = chunk.end();
6062 }
6063 _collectorState = Idling;
6064 } else {
6065 // already have the lock
6066 assert(_collectorState == Resetting, "just checking");
6067 assert_lock_strong(bitMapLock());
6068 _markBitMap.clear_all();
6069 _collectorState = Idling;
6070 }
6072 // Stop incremental mode after a cycle completes, so that any future cycles
6073 // are triggered by allocation.
6074 stop_icms();
6076 NOT_PRODUCT(
6077 if (RotateCMSCollectionTypes) {
6078 _cmsGen->rotate_debug_collection_type();
6079 }
6080 )
6081 }
6083 void CMSCollector::do_CMS_operation(CMS_op_type op) {
6084 gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
6085 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
6086 TraceTime t("GC", PrintGC, !PrintGCDetails, gclog_or_tty);
6087 TraceCollectorStats tcs(counters());
6089 switch (op) {
6090 case CMS_op_checkpointRootsInitial: {
6091 checkpointRootsInitial(true); // asynch
6092 if (PrintGC) {
6093 _cmsGen->printOccupancy("initial-mark");
6094 }
6095 break;
6096 }
6097 case CMS_op_checkpointRootsFinal: {
6098 checkpointRootsFinal(true, // asynch
6099 false, // !clear_all_soft_refs
6100 false); // !init_mark_was_synchronous
6101 if (PrintGC) {
6102 _cmsGen->printOccupancy("remark");
6103 }
6104 break;
6105 }
6106 default:
6107 fatal("No such CMS_op");
6108 }
6109 }
6111 #ifndef PRODUCT
6112 size_t const CMSCollector::skip_header_HeapWords() {
6113 return FreeChunk::header_size();
6114 }
6116 // Try and collect here conditions that should hold when
6117 // CMS thread is exiting. The idea is that the foreground GC
6118 // thread should not be blocked if it wants to terminate
6119 // the CMS thread and yet continue to run the VM for a while
6120 // after that.
6121 void CMSCollector::verify_ok_to_terminate() const {
6122 assert(Thread::current()->is_ConcurrentGC_thread(),
6123 "should be called by CMS thread");
6124 assert(!_foregroundGCShouldWait, "should be false");
6125 // We could check here that all the various low-level locks
6126 // are not held by the CMS thread, but that is overkill; see
6127 // also CMSThread::verify_ok_to_terminate() where the CGC_lock
6128 // is checked.
6129 }
6130 #endif
6132 size_t CMSCollector::block_size_using_printezis_bits(HeapWord* addr) const {
6133 assert(_markBitMap.isMarked(addr) && _markBitMap.isMarked(addr + 1),
6134 "missing Printezis mark?");
6135 HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
6136 size_t size = pointer_delta(nextOneAddr + 1, addr);
6137 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6138 "alignment problem");
6139 assert(size >= 3, "Necessary for Printezis marks to work");
6140 return size;
6141 }
6143 // A variant of the above (block_size_using_printezis_bits()) except
6144 // that we return 0 if the P-bits are not yet set.
6145 size_t CMSCollector::block_size_if_printezis_bits(HeapWord* addr) const {
6146 if (_markBitMap.isMarked(addr)) {
6147 assert(_markBitMap.isMarked(addr + 1), "Missing Printezis bit?");
6148 HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
6149 size_t size = pointer_delta(nextOneAddr + 1, addr);
6150 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6151 "alignment problem");
6152 assert(size >= 3, "Necessary for Printezis marks to work");
6153 return size;
6154 } else {
6155 assert(!_markBitMap.isMarked(addr + 1), "Bit map inconsistency?");
6156 return 0;
6157 }
6158 }
6160 HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
6161 size_t sz = 0;
6162 oop p = (oop)addr;
6163 if (p->klass() != NULL && p->is_parsable()) {
6164 sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
6165 } else {
6166 sz = block_size_using_printezis_bits(addr);
6167 }
6168 assert(sz > 0, "size must be nonzero");
6169 HeapWord* next_block = addr + sz;
6170 HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block,
6171 CardTableModRefBS::card_size);
6172 assert(round_down((uintptr_t)addr, CardTableModRefBS::card_size) <
6173 round_down((uintptr_t)next_card, CardTableModRefBS::card_size),
6174 "must be different cards");
6175 return next_card;
6176 }
6179 // CMS Bit Map Wrapper /////////////////////////////////////////
6181 // Construct a CMS bit map infrastructure, but don't create the
6182 // bit vector itself. That is done by a separate call CMSBitMap::allocate()
6183 // further below.
6184 CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
6185 _bm(NULL,0),
6186 _shifter(shifter),
6187 _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true) : NULL)
6188 {
6189 _bmStartWord = 0;
6190 _bmWordSize = 0;
6191 }
6193 bool CMSBitMap::allocate(MemRegion mr) {
6194 _bmStartWord = mr.start();
6195 _bmWordSize = mr.word_size();
6196 ReservedSpace brs(ReservedSpace::allocation_align_size_up(
6197 (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1));
6198 if (!brs.is_reserved()) {
6199 warning("CMS bit map allocation failure");
6200 return false;
6201 }
6202 // For now we'll just commit all of the bit map up fromt.
6203 // Later on we'll try to be more parsimonious with swap.
6204 if (!_virtual_space.initialize(brs, brs.size())) {
6205 warning("CMS bit map backing store failure");
6206 return false;
6207 }
6208 assert(_virtual_space.committed_size() == brs.size(),
6209 "didn't reserve backing store for all of CMS bit map?");
6210 _bm.set_map((uintptr_t*)_virtual_space.low());
6211 assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
6212 _bmWordSize, "inconsistency in bit map sizing");
6213 _bm.set_size(_bmWordSize >> _shifter);
6215 // bm.clear(); // can we rely on getting zero'd memory? verify below
6216 assert(isAllClear(),
6217 "Expected zero'd memory from ReservedSpace constructor");
6218 assert(_bm.size() == heapWordDiffToOffsetDiff(sizeInWords()),
6219 "consistency check");
6220 return true;
6221 }
6223 void CMSBitMap::dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl) {
6224 HeapWord *next_addr, *end_addr, *last_addr;
6225 assert_locked();
6226 assert(covers(mr), "out-of-range error");
6227 // XXX assert that start and end are appropriately aligned
6228 for (next_addr = mr.start(), end_addr = mr.end();
6229 next_addr < end_addr; next_addr = last_addr) {
6230 MemRegion dirty_region = getAndClearMarkedRegion(next_addr, end_addr);
6231 last_addr = dirty_region.end();
6232 if (!dirty_region.is_empty()) {
6233 cl->do_MemRegion(dirty_region);
6234 } else {
6235 assert(last_addr == end_addr, "program logic");
6236 return;
6237 }
6238 }
6239 }
6241 #ifndef PRODUCT
6242 void CMSBitMap::assert_locked() const {
6243 CMSLockVerifier::assert_locked(lock());
6244 }
6246 bool CMSBitMap::covers(MemRegion mr) const {
6247 // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
6248 assert((size_t)_bm.size() == (_bmWordSize >> _shifter),
6249 "size inconsistency");
6250 return (mr.start() >= _bmStartWord) &&
6251 (mr.end() <= endWord());
6252 }
6254 bool CMSBitMap::covers(HeapWord* start, size_t size) const {
6255 return (start >= _bmStartWord && (start + size) <= endWord());
6256 }
6258 void CMSBitMap::verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) {
6259 // verify that there are no 1 bits in the interval [left, right)
6260 FalseBitMapClosure falseBitMapClosure;
6261 iterate(&falseBitMapClosure, left, right);
6262 }
6264 void CMSBitMap::region_invariant(MemRegion mr)
6265 {
6266 assert_locked();
6267 // mr = mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
6268 assert(!mr.is_empty(), "unexpected empty region");
6269 assert(covers(mr), "mr should be covered by bit map");
6270 // convert address range into offset range
6271 size_t start_ofs = heapWordToOffset(mr.start());
6272 // Make sure that end() is appropriately aligned
6273 assert(mr.end() == (HeapWord*)round_to((intptr_t)mr.end(),
6274 (1 << (_shifter+LogHeapWordSize))),
6275 "Misaligned mr.end()");
6276 size_t end_ofs = heapWordToOffset(mr.end());
6277 assert(end_ofs > start_ofs, "Should mark at least one bit");
6278 }
6280 #endif
6282 bool CMSMarkStack::allocate(size_t size) {
6283 // allocate a stack of the requisite depth
6284 ReservedSpace rs(ReservedSpace::allocation_align_size_up(
6285 size * sizeof(oop)));
6286 if (!rs.is_reserved()) {
6287 warning("CMSMarkStack allocation failure");
6288 return false;
6289 }
6290 if (!_virtual_space.initialize(rs, rs.size())) {
6291 warning("CMSMarkStack backing store failure");
6292 return false;
6293 }
6294 assert(_virtual_space.committed_size() == rs.size(),
6295 "didn't reserve backing store for all of CMS stack?");
6296 _base = (oop*)(_virtual_space.low());
6297 _index = 0;
6298 _capacity = size;
6299 NOT_PRODUCT(_max_depth = 0);
6300 return true;
6301 }
6303 // XXX FIX ME !!! In the MT case we come in here holding a
6304 // leaf lock. For printing we need to take a further lock
6305 // which has lower rank. We need to recallibrate the two
6306 // lock-ranks involved in order to be able to rpint the
6307 // messages below. (Or defer the printing to the caller.
6308 // For now we take the expedient path of just disabling the
6309 // messages for the problematic case.)
6310 void CMSMarkStack::expand() {
6311 assert(_capacity <= CMSMarkStackSizeMax, "stack bigger than permitted");
6312 if (_capacity == CMSMarkStackSizeMax) {
6313 if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
6314 // We print a warning message only once per CMS cycle.
6315 gclog_or_tty->print_cr(" (benign) Hit CMSMarkStack max size limit");
6316 }
6317 return;
6318 }
6319 // Double capacity if possible
6320 size_t new_capacity = MIN2(_capacity*2, CMSMarkStackSizeMax);
6321 // Do not give up existing stack until we have managed to
6322 // get the double capacity that we desired.
6323 ReservedSpace rs(ReservedSpace::allocation_align_size_up(
6324 new_capacity * sizeof(oop)));
6325 if (rs.is_reserved()) {
6326 // Release the backing store associated with old stack
6327 _virtual_space.release();
6328 // Reinitialize virtual space for new stack
6329 if (!_virtual_space.initialize(rs, rs.size())) {
6330 fatal("Not enough swap for expanded marking stack");
6331 }
6332 _base = (oop*)(_virtual_space.low());
6333 _index = 0;
6334 _capacity = new_capacity;
6335 } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
6336 // Failed to double capacity, continue;
6337 // we print a detail message only once per CMS cycle.
6338 gclog_or_tty->print(" (benign) Failed to expand marking stack from "SIZE_FORMAT"K to "
6339 SIZE_FORMAT"K",
6340 _capacity / K, new_capacity / K);
6341 }
6342 }
6345 // Closures
6346 // XXX: there seems to be a lot of code duplication here;
6347 // should refactor and consolidate common code.
6349 // This closure is used to mark refs into the CMS generation in
6350 // the CMS bit map. Called at the first checkpoint. This closure
6351 // assumes that we do not need to re-mark dirty cards; if the CMS
6352 // generation on which this is used is not an oldest (modulo perm gen)
6353 // generation then this will lose younger_gen cards!
6355 MarkRefsIntoClosure::MarkRefsIntoClosure(
6356 MemRegion span, CMSBitMap* bitMap, bool should_do_nmethods):
6357 _span(span),
6358 _bitMap(bitMap),
6359 _should_do_nmethods(should_do_nmethods)
6360 {
6361 assert(_ref_processor == NULL, "deliberately left NULL");
6362 assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
6363 }
6365 void MarkRefsIntoClosure::do_oop(oop* p) {
6366 // if p points into _span, then mark corresponding bit in _markBitMap
6367 oop thisOop = *p;
6368 if (thisOop != NULL) {
6369 assert(thisOop->is_oop(), "expected an oop");
6370 HeapWord* addr = (HeapWord*)thisOop;
6371 if (_span.contains(addr)) {
6372 // this should be made more efficient
6373 _bitMap->mark(addr);
6374 }
6375 }
6376 }
6378 // A variant of the above, used for CMS marking verification.
6379 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
6380 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm,
6381 bool should_do_nmethods):
6382 _span(span),
6383 _verification_bm(verification_bm),
6384 _cms_bm(cms_bm),
6385 _should_do_nmethods(should_do_nmethods) {
6386 assert(_ref_processor == NULL, "deliberately left NULL");
6387 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
6388 }
6390 void MarkRefsIntoVerifyClosure::do_oop(oop* p) {
6391 // if p points into _span, then mark corresponding bit in _markBitMap
6392 oop this_oop = *p;
6393 if (this_oop != NULL) {
6394 assert(this_oop->is_oop(), "expected an oop");
6395 HeapWord* addr = (HeapWord*)this_oop;
6396 if (_span.contains(addr)) {
6397 _verification_bm->mark(addr);
6398 if (!_cms_bm->isMarked(addr)) {
6399 oop(addr)->print();
6400 gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", addr);
6401 fatal("... aborting");
6402 }
6403 }
6404 }
6405 }
6407 //////////////////////////////////////////////////
6408 // MarkRefsIntoAndScanClosure
6409 //////////////////////////////////////////////////
6411 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
6412 ReferenceProcessor* rp,
6413 CMSBitMap* bit_map,
6414 CMSBitMap* mod_union_table,
6415 CMSMarkStack* mark_stack,
6416 CMSMarkStack* revisit_stack,
6417 CMSCollector* collector,
6418 bool should_yield,
6419 bool concurrent_precleaning):
6420 _collector(collector),
6421 _span(span),
6422 _bit_map(bit_map),
6423 _mark_stack(mark_stack),
6424 _pushAndMarkClosure(collector, span, rp, bit_map, mod_union_table,
6425 mark_stack, revisit_stack, concurrent_precleaning),
6426 _yield(should_yield),
6427 _concurrent_precleaning(concurrent_precleaning),
6428 _freelistLock(NULL)
6429 {
6430 _ref_processor = rp;
6431 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
6432 }
6434 // This closure is used to mark refs into the CMS generation at the
6435 // second (final) checkpoint, and to scan and transitively follow
6436 // the unmarked oops. It is also used during the concurrent precleaning
6437 // phase while scanning objects on dirty cards in the CMS generation.
6438 // The marks are made in the marking bit map and the marking stack is
6439 // used for keeping the (newly) grey objects during the scan.
6440 // The parallel version (Par_...) appears further below.
6441 void MarkRefsIntoAndScanClosure::do_oop(oop* p) {
6442 oop this_oop = *p;
6443 if (this_oop != NULL) {
6444 assert(this_oop->is_oop(), "expected an oop");
6445 HeapWord* addr = (HeapWord*)this_oop;
6446 assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
6447 assert(_collector->overflow_list_is_empty(), "should be empty");
6448 if (_span.contains(addr) &&
6449 !_bit_map->isMarked(addr)) {
6450 // mark bit map (object is now grey)
6451 _bit_map->mark(addr);
6452 // push on marking stack (stack should be empty), and drain the
6453 // stack by applying this closure to the oops in the oops popped
6454 // from the stack (i.e. blacken the grey objects)
6455 bool res = _mark_stack->push(this_oop);
6456 assert(res, "Should have space to push on empty stack");
6457 do {
6458 oop new_oop = _mark_stack->pop();
6459 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
6460 assert(new_oop->is_parsable(), "Found unparsable oop");
6461 assert(_bit_map->isMarked((HeapWord*)new_oop),
6462 "only grey objects on this stack");
6463 // iterate over the oops in this oop, marking and pushing
6464 // the ones in CMS heap (i.e. in _span).
6465 new_oop->oop_iterate(&_pushAndMarkClosure);
6466 // check if it's time to yield
6467 do_yield_check();
6468 } while (!_mark_stack->isEmpty() ||
6469 (!_concurrent_precleaning && take_from_overflow_list()));
6470 // if marking stack is empty, and we are not doing this
6471 // during precleaning, then check the overflow list
6472 }
6473 assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
6474 assert(_collector->overflow_list_is_empty(),
6475 "overflow list was drained above");
6476 // We could restore evacuated mark words, if any, used for
6477 // overflow list links here because the overflow list is
6478 // provably empty here. That would reduce the maximum
6479 // size requirements for preserved_{oop,mark}_stack.
6480 // But we'll just postpone it until we are all done
6481 // so we can just stream through.
6482 if (!_concurrent_precleaning && CMSOverflowEarlyRestoration) {
6483 _collector->restore_preserved_marks_if_any();
6484 assert(_collector->no_preserved_marks(), "No preserved marks");
6485 }
6486 assert(!CMSOverflowEarlyRestoration || _collector->no_preserved_marks(),
6487 "All preserved marks should have been restored above");
6488 }
6489 }
6491 void MarkRefsIntoAndScanClosure::do_yield_work() {
6492 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6493 "CMS thread should hold CMS token");
6494 assert_lock_strong(_freelistLock);
6495 assert_lock_strong(_bit_map->lock());
6496 // relinquish the free_list_lock and bitMaplock()
6497 _bit_map->lock()->unlock();
6498 _freelistLock->unlock();
6499 ConcurrentMarkSweepThread::desynchronize(true);
6500 ConcurrentMarkSweepThread::acknowledge_yield_request();
6501 _collector->stopTimer();
6502 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
6503 if (PrintCMSStatistics != 0) {
6504 _collector->incrementYields();
6505 }
6506 _collector->icms_wait();
6508 // See the comment in coordinator_yield()
6509 for (unsigned i = 0; i < CMSYieldSleepCount &&
6510 ConcurrentMarkSweepThread::should_yield() &&
6511 !CMSCollector::foregroundGCIsActive(); ++i) {
6512 os::sleep(Thread::current(), 1, false);
6513 ConcurrentMarkSweepThread::acknowledge_yield_request();
6514 }
6516 ConcurrentMarkSweepThread::synchronize(true);
6517 _freelistLock->lock_without_safepoint_check();
6518 _bit_map->lock()->lock_without_safepoint_check();
6519 _collector->startTimer();
6520 }
6522 ///////////////////////////////////////////////////////////
6523 // Par_MarkRefsIntoAndScanClosure: a parallel version of
6524 // MarkRefsIntoAndScanClosure
6525 ///////////////////////////////////////////////////////////
6526 Par_MarkRefsIntoAndScanClosure::Par_MarkRefsIntoAndScanClosure(
6527 CMSCollector* collector, MemRegion span, ReferenceProcessor* rp,
6528 CMSBitMap* bit_map, OopTaskQueue* work_queue, CMSMarkStack* revisit_stack):
6529 _span(span),
6530 _bit_map(bit_map),
6531 _work_queue(work_queue),
6532 _low_water_mark(MIN2((uint)(work_queue->max_elems()/4),
6533 (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads))),
6534 _par_pushAndMarkClosure(collector, span, rp, bit_map, work_queue,
6535 revisit_stack)
6536 {
6537 _ref_processor = rp;
6538 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
6539 }
6541 // This closure is used to mark refs into the CMS generation at the
6542 // second (final) checkpoint, and to scan and transitively follow
6543 // the unmarked oops. The marks are made in the marking bit map and
6544 // the work_queue is used for keeping the (newly) grey objects during
6545 // the scan phase whence they are also available for stealing by parallel
6546 // threads. Since the marking bit map is shared, updates are
6547 // synchronized (via CAS).
6548 void Par_MarkRefsIntoAndScanClosure::do_oop(oop* p) {
6549 oop this_oop = *p;
6550 if (this_oop != NULL) {
6551 // Ignore mark word because this could be an already marked oop
6552 // that may be chained at the end of the overflow list.
6553 assert(this_oop->is_oop(true /* ignore mark word */), "expected an oop");
6554 HeapWord* addr = (HeapWord*)this_oop;
6555 if (_span.contains(addr) &&
6556 !_bit_map->isMarked(addr)) {
6557 // mark bit map (object will become grey):
6558 // It is possible for several threads to be
6559 // trying to "claim" this object concurrently;
6560 // the unique thread that succeeds in marking the
6561 // object first will do the subsequent push on
6562 // to the work queue (or overflow list).
6563 if (_bit_map->par_mark(addr)) {
6564 // push on work_queue (which may not be empty), and trim the
6565 // queue to an appropriate length by applying this closure to
6566 // the oops in the oops popped from the stack (i.e. blacken the
6567 // grey objects)
6568 bool res = _work_queue->push(this_oop);
6569 assert(res, "Low water mark should be less than capacity?");
6570 trim_queue(_low_water_mark);
6571 } // Else, another thread claimed the object
6572 }
6573 }
6574 }
6576 // This closure is used to rescan the marked objects on the dirty cards
6577 // in the mod union table and the card table proper.
6578 size_t ScanMarkedObjectsAgainCarefullyClosure::do_object_careful_m(
6579 oop p, MemRegion mr) {
6581 size_t size = 0;
6582 HeapWord* addr = (HeapWord*)p;
6583 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6584 assert(_span.contains(addr), "we are scanning the CMS generation");
6585 // check if it's time to yield
6586 if (do_yield_check()) {
6587 // We yielded for some foreground stop-world work,
6588 // and we have been asked to abort this ongoing preclean cycle.
6589 return 0;
6590 }
6591 if (_bitMap->isMarked(addr)) {
6592 // it's marked; is it potentially uninitialized?
6593 if (p->klass() != NULL) {
6594 if (CMSPermGenPrecleaningEnabled && !p->is_parsable()) {
6595 // Signal precleaning to redirty the card since
6596 // the klass pointer is already installed.
6597 assert(size == 0, "Initial value");
6598 } else {
6599 assert(p->is_parsable(), "must be parsable.");
6600 // an initialized object; ignore mark word in verification below
6601 // since we are running concurrent with mutators
6602 assert(p->is_oop(true), "should be an oop");
6603 if (p->is_objArray()) {
6604 // objArrays are precisely marked; restrict scanning
6605 // to dirty cards only.
6606 size = p->oop_iterate(_scanningClosure, mr);
6607 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6608 "adjustObjectSize should be the identity for array sizes, "
6609 "which are necessarily larger than minimum object size of "
6610 "two heap words");
6611 } else {
6612 // A non-array may have been imprecisely marked; we need
6613 // to scan object in its entirety.
6614 size = CompactibleFreeListSpace::adjustObjectSize(
6615 p->oop_iterate(_scanningClosure));
6616 }
6617 #ifdef DEBUG
6618 size_t direct_size =
6619 CompactibleFreeListSpace::adjustObjectSize(p->size());
6620 assert(size == direct_size, "Inconsistency in size");
6621 assert(size >= 3, "Necessary for Printezis marks to work");
6622 if (!_bitMap->isMarked(addr+1)) {
6623 _bitMap->verifyNoOneBitsInRange(addr+2, addr+size);
6624 } else {
6625 _bitMap->verifyNoOneBitsInRange(addr+2, addr+size-1);
6626 assert(_bitMap->isMarked(addr+size-1),
6627 "inconsistent Printezis mark");
6628 }
6629 #endif // DEBUG
6630 }
6631 } else {
6632 // an unitialized object
6633 assert(_bitMap->isMarked(addr+1), "missing Printezis mark?");
6634 HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
6635 size = pointer_delta(nextOneAddr + 1, addr);
6636 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6637 "alignment problem");
6638 // Note that pre-cleaning needn't redirty the card. OopDesc::set_klass()
6639 // will dirty the card when the klass pointer is installed in the
6640 // object (signalling the completion of initialization).
6641 }
6642 } else {
6643 // Either a not yet marked object or an uninitialized object
6644 if (p->klass() == NULL || !p->is_parsable()) {
6645 // An uninitialized object, skip to the next card, since
6646 // we may not be able to read its P-bits yet.
6647 assert(size == 0, "Initial value");
6648 } else {
6649 // An object not (yet) reached by marking: we merely need to
6650 // compute its size so as to go look at the next block.
6651 assert(p->is_oop(true), "should be an oop");
6652 size = CompactibleFreeListSpace::adjustObjectSize(p->size());
6653 }
6654 }
6655 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6656 return size;
6657 }
6659 void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
6660 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6661 "CMS thread should hold CMS token");
6662 assert_lock_strong(_freelistLock);
6663 assert_lock_strong(_bitMap->lock());
6664 // relinquish the free_list_lock and bitMaplock()
6665 _bitMap->lock()->unlock();
6666 _freelistLock->unlock();
6667 ConcurrentMarkSweepThread::desynchronize(true);
6668 ConcurrentMarkSweepThread::acknowledge_yield_request();
6669 _collector->stopTimer();
6670 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
6671 if (PrintCMSStatistics != 0) {
6672 _collector->incrementYields();
6673 }
6674 _collector->icms_wait();
6676 // See the comment in coordinator_yield()
6677 for (unsigned i = 0; i < CMSYieldSleepCount &&
6678 ConcurrentMarkSweepThread::should_yield() &&
6679 !CMSCollector::foregroundGCIsActive(); ++i) {
6680 os::sleep(Thread::current(), 1, false);
6681 ConcurrentMarkSweepThread::acknowledge_yield_request();
6682 }
6684 ConcurrentMarkSweepThread::synchronize(true);
6685 _freelistLock->lock_without_safepoint_check();
6686 _bitMap->lock()->lock_without_safepoint_check();
6687 _collector->startTimer();
6688 }
6691 //////////////////////////////////////////////////////////////////
6692 // SurvivorSpacePrecleanClosure
6693 //////////////////////////////////////////////////////////////////
6694 // This (single-threaded) closure is used to preclean the oops in
6695 // the survivor spaces.
6696 size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) {
6698 HeapWord* addr = (HeapWord*)p;
6699 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6700 assert(!_span.contains(addr), "we are scanning the survivor spaces");
6701 assert(p->klass() != NULL, "object should be initializd");
6702 assert(p->is_parsable(), "must be parsable.");
6703 // an initialized object; ignore mark word in verification below
6704 // since we are running concurrent with mutators
6705 assert(p->is_oop(true), "should be an oop");
6706 // Note that we do not yield while we iterate over
6707 // the interior oops of p, pushing the relevant ones
6708 // on our marking stack.
6709 size_t size = p->oop_iterate(_scanning_closure);
6710 do_yield_check();
6711 // Observe that below, we do not abandon the preclean
6712 // phase as soon as we should; rather we empty the
6713 // marking stack before returning. This is to satisfy
6714 // some existing assertions. In general, it may be a
6715 // good idea to abort immediately and complete the marking
6716 // from the grey objects at a later time.
6717 while (!_mark_stack->isEmpty()) {
6718 oop new_oop = _mark_stack->pop();
6719 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
6720 assert(new_oop->is_parsable(), "Found unparsable oop");
6721 assert(_bit_map->isMarked((HeapWord*)new_oop),
6722 "only grey objects on this stack");
6723 // iterate over the oops in this oop, marking and pushing
6724 // the ones in CMS heap (i.e. in _span).
6725 new_oop->oop_iterate(_scanning_closure);
6726 // check if it's time to yield
6727 do_yield_check();
6728 }
6729 unsigned int after_count =
6730 GenCollectedHeap::heap()->total_collections();
6731 bool abort = (_before_count != after_count) ||
6732 _collector->should_abort_preclean();
6733 return abort ? 0 : size;
6734 }
6736 void SurvivorSpacePrecleanClosure::do_yield_work() {
6737 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6738 "CMS thread should hold CMS token");
6739 assert_lock_strong(_bit_map->lock());
6740 // Relinquish the bit map lock
6741 _bit_map->lock()->unlock();
6742 ConcurrentMarkSweepThread::desynchronize(true);
6743 ConcurrentMarkSweepThread::acknowledge_yield_request();
6744 _collector->stopTimer();
6745 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
6746 if (PrintCMSStatistics != 0) {
6747 _collector->incrementYields();
6748 }
6749 _collector->icms_wait();
6751 // See the comment in coordinator_yield()
6752 for (unsigned i = 0; i < CMSYieldSleepCount &&
6753 ConcurrentMarkSweepThread::should_yield() &&
6754 !CMSCollector::foregroundGCIsActive(); ++i) {
6755 os::sleep(Thread::current(), 1, false);
6756 ConcurrentMarkSweepThread::acknowledge_yield_request();
6757 }
6759 ConcurrentMarkSweepThread::synchronize(true);
6760 _bit_map->lock()->lock_without_safepoint_check();
6761 _collector->startTimer();
6762 }
6764 // This closure is used to rescan the marked objects on the dirty cards
6765 // in the mod union table and the card table proper. In the parallel
6766 // case, although the bitMap is shared, we do a single read so the
6767 // isMarked() query is "safe".
6768 bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) {
6769 // Ignore mark word because we are running concurrent with mutators
6770 assert(p->is_oop_or_null(true), "expected an oop or null");
6771 HeapWord* addr = (HeapWord*)p;
6772 assert(_span.contains(addr), "we are scanning the CMS generation");
6773 bool is_obj_array = false;
6774 #ifdef DEBUG
6775 if (!_parallel) {
6776 assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)");
6777 assert(_collector->overflow_list_is_empty(),
6778 "overflow list should be empty");
6780 }
6781 #endif // DEBUG
6782 if (_bit_map->isMarked(addr)) {
6783 // Obj arrays are precisely marked, non-arrays are not;
6784 // so we scan objArrays precisely and non-arrays in their
6785 // entirety.
6786 if (p->is_objArray()) {
6787 is_obj_array = true;
6788 if (_parallel) {
6789 p->oop_iterate(_par_scan_closure, mr);
6790 } else {
6791 p->oop_iterate(_scan_closure, mr);
6792 }
6793 } else {
6794 if (_parallel) {
6795 p->oop_iterate(_par_scan_closure);
6796 } else {
6797 p->oop_iterate(_scan_closure);
6798 }
6799 }
6800 }
6801 #ifdef DEBUG
6802 if (!_parallel) {
6803 assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
6804 assert(_collector->overflow_list_is_empty(),
6805 "overflow list should be empty");
6807 }
6808 #endif // DEBUG
6809 return is_obj_array;
6810 }
6812 MarkFromRootsClosure::MarkFromRootsClosure(CMSCollector* collector,
6813 MemRegion span,
6814 CMSBitMap* bitMap, CMSMarkStack* markStack,
6815 CMSMarkStack* revisitStack,
6816 bool should_yield, bool verifying):
6817 _collector(collector),
6818 _span(span),
6819 _bitMap(bitMap),
6820 _mut(&collector->_modUnionTable),
6821 _markStack(markStack),
6822 _revisitStack(revisitStack),
6823 _yield(should_yield),
6824 _skipBits(0)
6825 {
6826 assert(_markStack->isEmpty(), "stack should be empty");
6827 _finger = _bitMap->startWord();
6828 _threshold = _finger;
6829 assert(_collector->_restart_addr == NULL, "Sanity check");
6830 assert(_span.contains(_finger), "Out of bounds _finger?");
6831 DEBUG_ONLY(_verifying = verifying;)
6832 }
6834 void MarkFromRootsClosure::reset(HeapWord* addr) {
6835 assert(_markStack->isEmpty(), "would cause duplicates on stack");
6836 assert(_span.contains(addr), "Out of bounds _finger?");
6837 _finger = addr;
6838 _threshold = (HeapWord*)round_to(
6839 (intptr_t)_finger, CardTableModRefBS::card_size);
6840 }
6842 // Should revisit to see if this should be restructured for
6843 // greater efficiency.
6844 void MarkFromRootsClosure::do_bit(size_t offset) {
6845 if (_skipBits > 0) {
6846 _skipBits--;
6847 return;
6848 }
6849 // convert offset into a HeapWord*
6850 HeapWord* addr = _bitMap->startWord() + offset;
6851 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
6852 "address out of range");
6853 assert(_bitMap->isMarked(addr), "tautology");
6854 if (_bitMap->isMarked(addr+1)) {
6855 // this is an allocated but not yet initialized object
6856 assert(_skipBits == 0, "tautology");
6857 _skipBits = 2; // skip next two marked bits ("Printezis-marks")
6858 oop p = oop(addr);
6859 if (p->klass() == NULL || !p->is_parsable()) {
6860 DEBUG_ONLY(if (!_verifying) {)
6861 // We re-dirty the cards on which this object lies and increase
6862 // the _threshold so that we'll come back to scan this object
6863 // during the preclean or remark phase. (CMSCleanOnEnter)
6864 if (CMSCleanOnEnter) {
6865 size_t sz = _collector->block_size_using_printezis_bits(addr);
6866 HeapWord* start_card_addr = (HeapWord*)round_down(
6867 (intptr_t)addr, CardTableModRefBS::card_size);
6868 HeapWord* end_card_addr = (HeapWord*)round_to(
6869 (intptr_t)(addr+sz), CardTableModRefBS::card_size);
6870 MemRegion redirty_range = MemRegion(start_card_addr, end_card_addr);
6871 assert(!redirty_range.is_empty(), "Arithmetical tautology");
6872 // Bump _threshold to end_card_addr; note that
6873 // _threshold cannot possibly exceed end_card_addr, anyhow.
6874 // This prevents future clearing of the card as the scan proceeds
6875 // to the right.
6876 assert(_threshold <= end_card_addr,
6877 "Because we are just scanning into this object");
6878 if (_threshold < end_card_addr) {
6879 _threshold = end_card_addr;
6880 }
6881 if (p->klass() != NULL) {
6882 // Redirty the range of cards...
6883 _mut->mark_range(redirty_range);
6884 } // ...else the setting of klass will dirty the card anyway.
6885 }
6886 DEBUG_ONLY(})
6887 return;
6888 }
6889 }
6890 scanOopsInOop(addr);
6891 }
6893 // We take a break if we've been at this for a while,
6894 // so as to avoid monopolizing the locks involved.
6895 void MarkFromRootsClosure::do_yield_work() {
6896 // First give up the locks, then yield, then re-lock
6897 // We should probably use a constructor/destructor idiom to
6898 // do this unlock/lock or modify the MutexUnlocker class to
6899 // serve our purpose. XXX
6900 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6901 "CMS thread should hold CMS token");
6902 assert_lock_strong(_bitMap->lock());
6903 _bitMap->lock()->unlock();
6904 ConcurrentMarkSweepThread::desynchronize(true);
6905 ConcurrentMarkSweepThread::acknowledge_yield_request();
6906 _collector->stopTimer();
6907 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
6908 if (PrintCMSStatistics != 0) {
6909 _collector->incrementYields();
6910 }
6911 _collector->icms_wait();
6913 // See the comment in coordinator_yield()
6914 for (unsigned i = 0; i < CMSYieldSleepCount &&
6915 ConcurrentMarkSweepThread::should_yield() &&
6916 !CMSCollector::foregroundGCIsActive(); ++i) {
6917 os::sleep(Thread::current(), 1, false);
6918 ConcurrentMarkSweepThread::acknowledge_yield_request();
6919 }
6921 ConcurrentMarkSweepThread::synchronize(true);
6922 _bitMap->lock()->lock_without_safepoint_check();
6923 _collector->startTimer();
6924 }
6926 void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) {
6927 assert(_bitMap->isMarked(ptr), "expected bit to be set");
6928 assert(_markStack->isEmpty(),
6929 "should drain stack to limit stack usage");
6930 // convert ptr to an oop preparatory to scanning
6931 oop this_oop = oop(ptr);
6932 // Ignore mark word in verification below, since we
6933 // may be running concurrent with mutators.
6934 assert(this_oop->is_oop(true), "should be an oop");
6935 assert(_finger <= ptr, "_finger runneth ahead");
6936 // advance the finger to right end of this object
6937 _finger = ptr + this_oop->size();
6938 assert(_finger > ptr, "we just incremented it above");
6939 // On large heaps, it may take us some time to get through
6940 // the marking phase (especially if running iCMS). During
6941 // this time it's possible that a lot of mutations have
6942 // accumulated in the card table and the mod union table --
6943 // these mutation records are redundant until we have
6944 // actually traced into the corresponding card.
6945 // Here, we check whether advancing the finger would make
6946 // us cross into a new card, and if so clear corresponding
6947 // cards in the MUT (preclean them in the card-table in the
6948 // future).
6950 DEBUG_ONLY(if (!_verifying) {)
6951 // The clean-on-enter optimization is disabled by default,
6952 // until we fix 6178663.
6953 if (CMSCleanOnEnter && (_finger > _threshold)) {
6954 // [_threshold, _finger) represents the interval
6955 // of cards to be cleared in MUT (or precleaned in card table).
6956 // The set of cards to be cleared is all those that overlap
6957 // with the interval [_threshold, _finger); note that
6958 // _threshold is always kept card-aligned but _finger isn't
6959 // always card-aligned.
6960 HeapWord* old_threshold = _threshold;
6961 assert(old_threshold == (HeapWord*)round_to(
6962 (intptr_t)old_threshold, CardTableModRefBS::card_size),
6963 "_threshold should always be card-aligned");
6964 _threshold = (HeapWord*)round_to(
6965 (intptr_t)_finger, CardTableModRefBS::card_size);
6966 MemRegion mr(old_threshold, _threshold);
6967 assert(!mr.is_empty(), "Control point invariant");
6968 assert(_span.contains(mr), "Should clear within span");
6969 // XXX When _finger crosses from old gen into perm gen
6970 // we may be doing unnecessary cleaning; do better in the
6971 // future by detecting that condition and clearing fewer
6972 // MUT/CT entries.
6973 _mut->clear_range(mr);
6974 }
6975 DEBUG_ONLY(})
6977 // Note: the finger doesn't advance while we drain
6978 // the stack below.
6979 PushOrMarkClosure pushOrMarkClosure(_collector,
6980 _span, _bitMap, _markStack,
6981 _revisitStack,
6982 _finger, this);
6983 bool res = _markStack->push(this_oop);
6984 assert(res, "Empty non-zero size stack should have space for single push");
6985 while (!_markStack->isEmpty()) {
6986 oop new_oop = _markStack->pop();
6987 // Skip verifying header mark word below because we are
6988 // running concurrent with mutators.
6989 assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
6990 // now scan this oop's oops
6991 new_oop->oop_iterate(&pushOrMarkClosure);
6992 do_yield_check();
6993 }
6994 assert(_markStack->isEmpty(), "tautology, emphasizing post-condition");
6995 }
6997 Par_MarkFromRootsClosure::Par_MarkFromRootsClosure(CMSConcMarkingTask* task,
6998 CMSCollector* collector, MemRegion span,
6999 CMSBitMap* bit_map,
7000 OopTaskQueue* work_queue,
7001 CMSMarkStack* overflow_stack,
7002 CMSMarkStack* revisit_stack,
7003 bool should_yield):
7004 _collector(collector),
7005 _whole_span(collector->_span),
7006 _span(span),
7007 _bit_map(bit_map),
7008 _mut(&collector->_modUnionTable),
7009 _work_queue(work_queue),
7010 _overflow_stack(overflow_stack),
7011 _revisit_stack(revisit_stack),
7012 _yield(should_yield),
7013 _skip_bits(0),
7014 _task(task)
7015 {
7016 assert(_work_queue->size() == 0, "work_queue should be empty");
7017 _finger = span.start();
7018 _threshold = _finger; // XXX Defer clear-on-enter optimization for now
7019 assert(_span.contains(_finger), "Out of bounds _finger?");
7020 }
7022 // Should revisit to see if this should be restructured for
7023 // greater efficiency.
7024 void Par_MarkFromRootsClosure::do_bit(size_t offset) {
7025 if (_skip_bits > 0) {
7026 _skip_bits--;
7027 return;
7028 }
7029 // convert offset into a HeapWord*
7030 HeapWord* addr = _bit_map->startWord() + offset;
7031 assert(_bit_map->endWord() && addr < _bit_map->endWord(),
7032 "address out of range");
7033 assert(_bit_map->isMarked(addr), "tautology");
7034 if (_bit_map->isMarked(addr+1)) {
7035 // this is an allocated object that might not yet be initialized
7036 assert(_skip_bits == 0, "tautology");
7037 _skip_bits = 2; // skip next two marked bits ("Printezis-marks")
7038 oop p = oop(addr);
7039 if (p->klass() == NULL || !p->is_parsable()) {
7040 // in the case of Clean-on-Enter optimization, redirty card
7041 // and avoid clearing card by increasing the threshold.
7042 return;
7043 }
7044 }
7045 scan_oops_in_oop(addr);
7046 }
7048 void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) {
7049 assert(_bit_map->isMarked(ptr), "expected bit to be set");
7050 // Should we assert that our work queue is empty or
7051 // below some drain limit?
7052 assert(_work_queue->size() == 0,
7053 "should drain stack to limit stack usage");
7054 // convert ptr to an oop preparatory to scanning
7055 oop this_oop = oop(ptr);
7056 // Ignore mark word in verification below, since we
7057 // may be running concurrent with mutators.
7058 assert(this_oop->is_oop(true), "should be an oop");
7059 assert(_finger <= ptr, "_finger runneth ahead");
7060 // advance the finger to right end of this object
7061 _finger = ptr + this_oop->size();
7062 assert(_finger > ptr, "we just incremented it above");
7063 // On large heaps, it may take us some time to get through
7064 // the marking phase (especially if running iCMS). During
7065 // this time it's possible that a lot of mutations have
7066 // accumulated in the card table and the mod union table --
7067 // these mutation records are redundant until we have
7068 // actually traced into the corresponding card.
7069 // Here, we check whether advancing the finger would make
7070 // us cross into a new card, and if so clear corresponding
7071 // cards in the MUT (preclean them in the card-table in the
7072 // future).
7074 // The clean-on-enter optimization is disabled by default,
7075 // until we fix 6178663.
7076 if (CMSCleanOnEnter && (_finger > _threshold)) {
7077 // [_threshold, _finger) represents the interval
7078 // of cards to be cleared in MUT (or precleaned in card table).
7079 // The set of cards to be cleared is all those that overlap
7080 // with the interval [_threshold, _finger); note that
7081 // _threshold is always kept card-aligned but _finger isn't
7082 // always card-aligned.
7083 HeapWord* old_threshold = _threshold;
7084 assert(old_threshold == (HeapWord*)round_to(
7085 (intptr_t)old_threshold, CardTableModRefBS::card_size),
7086 "_threshold should always be card-aligned");
7087 _threshold = (HeapWord*)round_to(
7088 (intptr_t)_finger, CardTableModRefBS::card_size);
7089 MemRegion mr(old_threshold, _threshold);
7090 assert(!mr.is_empty(), "Control point invariant");
7091 assert(_span.contains(mr), "Should clear within span"); // _whole_span ??
7092 // XXX When _finger crosses from old gen into perm gen
7093 // we may be doing unnecessary cleaning; do better in the
7094 // future by detecting that condition and clearing fewer
7095 // MUT/CT entries.
7096 _mut->clear_range(mr);
7097 }
7099 // Note: the local finger doesn't advance while we drain
7100 // the stack below, but the global finger sure can and will.
7101 HeapWord** gfa = _task->global_finger_addr();
7102 Par_PushOrMarkClosure pushOrMarkClosure(_collector,
7103 _span, _bit_map,
7104 _work_queue,
7105 _overflow_stack,
7106 _revisit_stack,
7107 _finger,
7108 gfa, this);
7109 bool res = _work_queue->push(this_oop); // overflow could occur here
7110 assert(res, "Will hold once we use workqueues");
7111 while (true) {
7112 oop new_oop;
7113 if (!_work_queue->pop_local(new_oop)) {
7114 // We emptied our work_queue; check if there's stuff that can
7115 // be gotten from the overflow stack.
7116 if (CMSConcMarkingTask::get_work_from_overflow_stack(
7117 _overflow_stack, _work_queue)) {
7118 do_yield_check();
7119 continue;
7120 } else { // done
7121 break;
7122 }
7123 }
7124 // Skip verifying header mark word below because we are
7125 // running concurrent with mutators.
7126 assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
7127 // now scan this oop's oops
7128 new_oop->oop_iterate(&pushOrMarkClosure);
7129 do_yield_check();
7130 }
7131 assert(_work_queue->size() == 0, "tautology, emphasizing post-condition");
7132 }
7134 // Yield in response to a request from VM Thread or
7135 // from mutators.
7136 void Par_MarkFromRootsClosure::do_yield_work() {
7137 assert(_task != NULL, "sanity");
7138 _task->yield();
7139 }
7141 // A variant of the above used for verifying CMS marking work.
7142 MarkFromRootsVerifyClosure::MarkFromRootsVerifyClosure(CMSCollector* collector,
7143 MemRegion span,
7144 CMSBitMap* verification_bm, CMSBitMap* cms_bm,
7145 CMSMarkStack* mark_stack):
7146 _collector(collector),
7147 _span(span),
7148 _verification_bm(verification_bm),
7149 _cms_bm(cms_bm),
7150 _mark_stack(mark_stack),
7151 _pam_verify_closure(collector, span, verification_bm, cms_bm,
7152 mark_stack)
7153 {
7154 assert(_mark_stack->isEmpty(), "stack should be empty");
7155 _finger = _verification_bm->startWord();
7156 assert(_collector->_restart_addr == NULL, "Sanity check");
7157 assert(_span.contains(_finger), "Out of bounds _finger?");
7158 }
7160 void MarkFromRootsVerifyClosure::reset(HeapWord* addr) {
7161 assert(_mark_stack->isEmpty(), "would cause duplicates on stack");
7162 assert(_span.contains(addr), "Out of bounds _finger?");
7163 _finger = addr;
7164 }
7166 // Should revisit to see if this should be restructured for
7167 // greater efficiency.
7168 void MarkFromRootsVerifyClosure::do_bit(size_t offset) {
7169 // convert offset into a HeapWord*
7170 HeapWord* addr = _verification_bm->startWord() + offset;
7171 assert(_verification_bm->endWord() && addr < _verification_bm->endWord(),
7172 "address out of range");
7173 assert(_verification_bm->isMarked(addr), "tautology");
7174 assert(_cms_bm->isMarked(addr), "tautology");
7176 assert(_mark_stack->isEmpty(),
7177 "should drain stack to limit stack usage");
7178 // convert addr to an oop preparatory to scanning
7179 oop this_oop = oop(addr);
7180 assert(this_oop->is_oop(), "should be an oop");
7181 assert(_finger <= addr, "_finger runneth ahead");
7182 // advance the finger to right end of this object
7183 _finger = addr + this_oop->size();
7184 assert(_finger > addr, "we just incremented it above");
7185 // Note: the finger doesn't advance while we drain
7186 // the stack below.
7187 bool res = _mark_stack->push(this_oop);
7188 assert(res, "Empty non-zero size stack should have space for single push");
7189 while (!_mark_stack->isEmpty()) {
7190 oop new_oop = _mark_stack->pop();
7191 assert(new_oop->is_oop(), "Oops! expected to pop an oop");
7192 // now scan this oop's oops
7193 new_oop->oop_iterate(&_pam_verify_closure);
7194 }
7195 assert(_mark_stack->isEmpty(), "tautology, emphasizing post-condition");
7196 }
7198 PushAndMarkVerifyClosure::PushAndMarkVerifyClosure(
7199 CMSCollector* collector, MemRegion span,
7200 CMSBitMap* verification_bm, CMSBitMap* cms_bm,
7201 CMSMarkStack* mark_stack):
7202 OopClosure(collector->ref_processor()),
7203 _collector(collector),
7204 _span(span),
7205 _verification_bm(verification_bm),
7206 _cms_bm(cms_bm),
7207 _mark_stack(mark_stack)
7208 { }
7211 // Upon stack overflow, we discard (part of) the stack,
7212 // remembering the least address amongst those discarded
7213 // in CMSCollector's _restart_address.
7214 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
7215 // Remember the least grey address discarded
7216 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
7217 _collector->lower_restart_addr(ra);
7218 _mark_stack->reset(); // discard stack contents
7219 _mark_stack->expand(); // expand the stack if possible
7220 }
7222 void PushAndMarkVerifyClosure::do_oop(oop* p) {
7223 oop this_oop = *p;
7224 assert(this_oop->is_oop_or_null(), "expected an oop or NULL");
7225 HeapWord* addr = (HeapWord*)this_oop;
7226 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
7227 // Oop lies in _span and isn't yet grey or black
7228 _verification_bm->mark(addr); // now grey
7229 if (!_cms_bm->isMarked(addr)) {
7230 oop(addr)->print();
7231 gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", addr);
7232 fatal("... aborting");
7233 }
7235 if (!_mark_stack->push(this_oop)) { // stack overflow
7236 if (PrintCMSStatistics != 0) {
7237 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
7238 SIZE_FORMAT, _mark_stack->capacity());
7239 }
7240 assert(_mark_stack->isFull(), "Else push should have succeeded");
7241 handle_stack_overflow(addr);
7242 }
7243 // anything including and to the right of _finger
7244 // will be scanned as we iterate over the remainder of the
7245 // bit map
7246 }
7247 }
7249 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
7250 MemRegion span,
7251 CMSBitMap* bitMap, CMSMarkStack* markStack,
7252 CMSMarkStack* revisitStack,
7253 HeapWord* finger, MarkFromRootsClosure* parent) :
7254 OopClosure(collector->ref_processor()),
7255 _collector(collector),
7256 _span(span),
7257 _bitMap(bitMap),
7258 _markStack(markStack),
7259 _revisitStack(revisitStack),
7260 _finger(finger),
7261 _parent(parent),
7262 _should_remember_klasses(collector->should_unload_classes())
7263 { }
7265 Par_PushOrMarkClosure::Par_PushOrMarkClosure(CMSCollector* collector,
7266 MemRegion span,
7267 CMSBitMap* bit_map,
7268 OopTaskQueue* work_queue,
7269 CMSMarkStack* overflow_stack,
7270 CMSMarkStack* revisit_stack,
7271 HeapWord* finger,
7272 HeapWord** global_finger_addr,
7273 Par_MarkFromRootsClosure* parent) :
7274 OopClosure(collector->ref_processor()),
7275 _collector(collector),
7276 _whole_span(collector->_span),
7277 _span(span),
7278 _bit_map(bit_map),
7279 _work_queue(work_queue),
7280 _overflow_stack(overflow_stack),
7281 _revisit_stack(revisit_stack),
7282 _finger(finger),
7283 _global_finger_addr(global_finger_addr),
7284 _parent(parent),
7285 _should_remember_klasses(collector->should_unload_classes())
7286 { }
7289 void CMSCollector::lower_restart_addr(HeapWord* low) {
7290 assert(_span.contains(low), "Out of bounds addr");
7291 if (_restart_addr == NULL) {
7292 _restart_addr = low;
7293 } else {
7294 _restart_addr = MIN2(_restart_addr, low);
7295 }
7296 }
7298 // Upon stack overflow, we discard (part of) the stack,
7299 // remembering the least address amongst those discarded
7300 // in CMSCollector's _restart_address.
7301 void PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
7302 // Remember the least grey address discarded
7303 HeapWord* ra = (HeapWord*)_markStack->least_value(lost);
7304 _collector->lower_restart_addr(ra);
7305 _markStack->reset(); // discard stack contents
7306 _markStack->expand(); // expand the stack if possible
7307 }
7309 // Upon stack overflow, we discard (part of) the stack,
7310 // remembering the least address amongst those discarded
7311 // in CMSCollector's _restart_address.
7312 void Par_PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
7313 // We need to do this under a mutex to prevent other
7314 // workers from interfering with the expansion below.
7315 MutexLockerEx ml(_overflow_stack->par_lock(),
7316 Mutex::_no_safepoint_check_flag);
7317 // Remember the least grey address discarded
7318 HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost);
7319 _collector->lower_restart_addr(ra);
7320 _overflow_stack->reset(); // discard stack contents
7321 _overflow_stack->expand(); // expand the stack if possible
7322 }
7325 void PushOrMarkClosure::do_oop(oop* p) {
7326 oop thisOop = *p;
7327 // Ignore mark word because we are running concurrent with mutators.
7328 assert(thisOop->is_oop_or_null(true), "expected an oop or NULL");
7329 HeapWord* addr = (HeapWord*)thisOop;
7330 if (_span.contains(addr) && !_bitMap->isMarked(addr)) {
7331 // Oop lies in _span and isn't yet grey or black
7332 _bitMap->mark(addr); // now grey
7333 if (addr < _finger) {
7334 // the bit map iteration has already either passed, or
7335 // sampled, this bit in the bit map; we'll need to
7336 // use the marking stack to scan this oop's oops.
7337 bool simulate_overflow = false;
7338 NOT_PRODUCT(
7339 if (CMSMarkStackOverflowALot &&
7340 _collector->simulate_overflow()) {
7341 // simulate a stack overflow
7342 simulate_overflow = true;
7343 }
7344 )
7345 if (simulate_overflow || !_markStack->push(thisOop)) { // stack overflow
7346 if (PrintCMSStatistics != 0) {
7347 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
7348 SIZE_FORMAT, _markStack->capacity());
7349 }
7350 assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded");
7351 handle_stack_overflow(addr);
7352 }
7353 }
7354 // anything including and to the right of _finger
7355 // will be scanned as we iterate over the remainder of the
7356 // bit map
7357 do_yield_check();
7358 }
7359 }
7361 void Par_PushOrMarkClosure::do_oop(oop* p) {
7362 oop this_oop = *p;
7363 // Ignore mark word because we are running concurrent with mutators.
7364 assert(this_oop->is_oop_or_null(true), "expected an oop or NULL");
7365 HeapWord* addr = (HeapWord*)this_oop;
7366 if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) {
7367 // Oop lies in _span and isn't yet grey or black
7368 // We read the global_finger (volatile read) strictly after marking oop
7369 bool res = _bit_map->par_mark(addr); // now grey
7370 volatile HeapWord** gfa = (volatile HeapWord**)_global_finger_addr;
7371 // Should we push this marked oop on our stack?
7372 // -- if someone else marked it, nothing to do
7373 // -- if target oop is above global finger nothing to do
7374 // -- if target oop is in chunk and above local finger
7375 // then nothing to do
7376 // -- else push on work queue
7377 if ( !res // someone else marked it, they will deal with it
7378 || (addr >= *gfa) // will be scanned in a later task
7379 || (_span.contains(addr) && addr >= _finger)) { // later in this chunk
7380 return;
7381 }
7382 // the bit map iteration has already either passed, or
7383 // sampled, this bit in the bit map; we'll need to
7384 // use the marking stack to scan this oop's oops.
7385 bool simulate_overflow = false;
7386 NOT_PRODUCT(
7387 if (CMSMarkStackOverflowALot &&
7388 _collector->simulate_overflow()) {
7389 // simulate a stack overflow
7390 simulate_overflow = true;
7391 }
7392 )
7393 if (simulate_overflow ||
7394 !(_work_queue->push(this_oop) || _overflow_stack->par_push(this_oop))) {
7395 // stack overflow
7396 if (PrintCMSStatistics != 0) {
7397 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
7398 SIZE_FORMAT, _overflow_stack->capacity());
7399 }
7400 // We cannot assert that the overflow stack is full because
7401 // it may have been emptied since.
7402 assert(simulate_overflow ||
7403 _work_queue->size() == _work_queue->max_elems(),
7404 "Else push should have succeeded");
7405 handle_stack_overflow(addr);
7406 }
7407 do_yield_check();
7408 }
7409 }
7412 PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector,
7413 MemRegion span,
7414 ReferenceProcessor* rp,
7415 CMSBitMap* bit_map,
7416 CMSBitMap* mod_union_table,
7417 CMSMarkStack* mark_stack,
7418 CMSMarkStack* revisit_stack,
7419 bool concurrent_precleaning):
7420 OopClosure(rp),
7421 _collector(collector),
7422 _span(span),
7423 _bit_map(bit_map),
7424 _mod_union_table(mod_union_table),
7425 _mark_stack(mark_stack),
7426 _revisit_stack(revisit_stack),
7427 _concurrent_precleaning(concurrent_precleaning),
7428 _should_remember_klasses(collector->should_unload_classes())
7429 {
7430 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
7431 }
7433 // Grey object rescan during pre-cleaning and second checkpoint phases --
7434 // the non-parallel version (the parallel version appears further below.)
7435 void PushAndMarkClosure::do_oop(oop* p) {
7436 oop this_oop = *p;
7437 // Ignore mark word verification. If during concurrent precleaning
7438 // the object monitor may be locked. If during the checkpoint
7439 // phases, the object may already have been reached by a different
7440 // path and may be at the end of the global overflow list (so
7441 // the mark word may be NULL).
7442 assert(this_oop->is_oop_or_null(true/* ignore mark word */),
7443 "expected an oop or NULL");
7444 HeapWord* addr = (HeapWord*)this_oop;
7445 // Check if oop points into the CMS generation
7446 // and is not marked
7447 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
7448 // a white object ...
7449 _bit_map->mark(addr); // ... now grey
7450 // push on the marking stack (grey set)
7451 bool simulate_overflow = false;
7452 NOT_PRODUCT(
7453 if (CMSMarkStackOverflowALot &&
7454 _collector->simulate_overflow()) {
7455 // simulate a stack overflow
7456 simulate_overflow = true;
7457 }
7458 )
7459 if (simulate_overflow || !_mark_stack->push(this_oop)) {
7460 if (_concurrent_precleaning) {
7461 // During precleaning we can just dirty the appropriate card
7462 // in the mod union table, thus ensuring that the object remains
7463 // in the grey set and continue. Note that no one can be intefering
7464 // with us in this action of dirtying the mod union table, so
7465 // no locking is required.
7466 _mod_union_table->mark(addr);
7467 _collector->_ser_pmc_preclean_ovflw++;
7468 } else {
7469 // During the remark phase, we need to remember this oop
7470 // in the overflow list.
7471 _collector->push_on_overflow_list(this_oop);
7472 _collector->_ser_pmc_remark_ovflw++;
7473 }
7474 }
7475 }
7476 }
7478 Par_PushAndMarkClosure::Par_PushAndMarkClosure(CMSCollector* collector,
7479 MemRegion span,
7480 ReferenceProcessor* rp,
7481 CMSBitMap* bit_map,
7482 OopTaskQueue* work_queue,
7483 CMSMarkStack* revisit_stack):
7484 OopClosure(rp),
7485 _collector(collector),
7486 _span(span),
7487 _bit_map(bit_map),
7488 _work_queue(work_queue),
7489 _revisit_stack(revisit_stack),
7490 _should_remember_klasses(collector->should_unload_classes())
7491 {
7492 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
7493 }
7495 // Grey object rescan during second checkpoint phase --
7496 // the parallel version.
7497 void Par_PushAndMarkClosure::do_oop(oop* p) {
7498 oop this_oop = *p;
7499 // In the assert below, we ignore the mark word because
7500 // this oop may point to an already visited object that is
7501 // on the overflow stack (in which case the mark word has
7502 // been hijacked for chaining into the overflow stack --
7503 // if this is the last object in the overflow stack then
7504 // its mark word will be NULL). Because this object may
7505 // have been subsequently popped off the global overflow
7506 // stack, and the mark word possibly restored to the prototypical
7507 // value, by the time we get to examined this failing assert in
7508 // the debugger, is_oop_or_null(false) may subsequently start
7509 // to hold.
7510 assert(this_oop->is_oop_or_null(true),
7511 "expected an oop or NULL");
7512 HeapWord* addr = (HeapWord*)this_oop;
7513 // Check if oop points into the CMS generation
7514 // and is not marked
7515 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
7516 // a white object ...
7517 // If we manage to "claim" the object, by being the
7518 // first thread to mark it, then we push it on our
7519 // marking stack
7520 if (_bit_map->par_mark(addr)) { // ... now grey
7521 // push on work queue (grey set)
7522 bool simulate_overflow = false;
7523 NOT_PRODUCT(
7524 if (CMSMarkStackOverflowALot &&
7525 _collector->par_simulate_overflow()) {
7526 // simulate a stack overflow
7527 simulate_overflow = true;
7528 }
7529 )
7530 if (simulate_overflow || !_work_queue->push(this_oop)) {
7531 _collector->par_push_on_overflow_list(this_oop);
7532 _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS
7533 }
7534 } // Else, some other thread got there first
7535 }
7536 }
7538 void PushAndMarkClosure::remember_klass(Klass* k) {
7539 if (!_revisit_stack->push(oop(k))) {
7540 fatal("Revisit stack overflowed in PushAndMarkClosure");
7541 }
7542 }
7544 void Par_PushAndMarkClosure::remember_klass(Klass* k) {
7545 if (!_revisit_stack->par_push(oop(k))) {
7546 fatal("Revist stack overflowed in Par_PushAndMarkClosure");
7547 }
7548 }
7550 void CMSPrecleanRefsYieldClosure::do_yield_work() {
7551 Mutex* bml = _collector->bitMapLock();
7552 assert_lock_strong(bml);
7553 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7554 "CMS thread should hold CMS token");
7556 bml->unlock();
7557 ConcurrentMarkSweepThread::desynchronize(true);
7559 ConcurrentMarkSweepThread::acknowledge_yield_request();
7561 _collector->stopTimer();
7562 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
7563 if (PrintCMSStatistics != 0) {
7564 _collector->incrementYields();
7565 }
7566 _collector->icms_wait();
7568 // See the comment in coordinator_yield()
7569 for (unsigned i = 0; i < CMSYieldSleepCount &&
7570 ConcurrentMarkSweepThread::should_yield() &&
7571 !CMSCollector::foregroundGCIsActive(); ++i) {
7572 os::sleep(Thread::current(), 1, false);
7573 ConcurrentMarkSweepThread::acknowledge_yield_request();
7574 }
7576 ConcurrentMarkSweepThread::synchronize(true);
7577 bml->lock();
7579 _collector->startTimer();
7580 }
7582 bool CMSPrecleanRefsYieldClosure::should_return() {
7583 if (ConcurrentMarkSweepThread::should_yield()) {
7584 do_yield_work();
7585 }
7586 return _collector->foregroundGCIsActive();
7587 }
7589 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
7590 assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
7591 "mr should be aligned to start at a card boundary");
7592 // We'd like to assert:
7593 // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
7594 // "mr should be a range of cards");
7595 // However, that would be too strong in one case -- the last
7596 // partition ends at _unallocated_block which, in general, can be
7597 // an arbitrary boundary, not necessarily card aligned.
7598 if (PrintCMSStatistics != 0) {
7599 _num_dirty_cards +=
7600 mr.word_size()/CardTableModRefBS::card_size_in_words;
7601 }
7602 _space->object_iterate_mem(mr, &_scan_cl);
7603 }
7605 SweepClosure::SweepClosure(CMSCollector* collector,
7606 ConcurrentMarkSweepGeneration* g,
7607 CMSBitMap* bitMap, bool should_yield) :
7608 _collector(collector),
7609 _g(g),
7610 _sp(g->cmsSpace()),
7611 _limit(_sp->sweep_limit()),
7612 _freelistLock(_sp->freelistLock()),
7613 _bitMap(bitMap),
7614 _yield(should_yield),
7615 _inFreeRange(false), // No free range at beginning of sweep
7616 _freeRangeInFreeLists(false), // No free range at beginning of sweep
7617 _lastFreeRangeCoalesced(false),
7618 _freeFinger(g->used_region().start())
7619 {
7620 NOT_PRODUCT(
7621 _numObjectsFreed = 0;
7622 _numWordsFreed = 0;
7623 _numObjectsLive = 0;
7624 _numWordsLive = 0;
7625 _numObjectsAlreadyFree = 0;
7626 _numWordsAlreadyFree = 0;
7627 _last_fc = NULL;
7629 _sp->initializeIndexedFreeListArrayReturnedBytes();
7630 _sp->dictionary()->initializeDictReturnedBytes();
7631 )
7632 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7633 "sweep _limit out of bounds");
7634 if (CMSTraceSweeper) {
7635 gclog_or_tty->print("\n====================\nStarting new sweep\n");
7636 }
7637 }
7639 // We need this destructor to reclaim any space at the end
7640 // of the space, which do_blk below may not have added back to
7641 // the free lists. [basically dealing with the "fringe effect"]
7642 SweepClosure::~SweepClosure() {
7643 assert_lock_strong(_freelistLock);
7644 // this should be treated as the end of a free run if any
7645 // The current free range should be returned to the free lists
7646 // as one coalesced chunk.
7647 if (inFreeRange()) {
7648 flushCurFreeChunk(freeFinger(),
7649 pointer_delta(_limit, freeFinger()));
7650 assert(freeFinger() < _limit, "the finger pointeth off base");
7651 if (CMSTraceSweeper) {
7652 gclog_or_tty->print("destructor:");
7653 gclog_or_tty->print("Sweep:put_free_blk 0x%x ("SIZE_FORMAT") "
7654 "[coalesced:"SIZE_FORMAT"]\n",
7655 freeFinger(), pointer_delta(_limit, freeFinger()),
7656 lastFreeRangeCoalesced());
7657 }
7658 }
7659 NOT_PRODUCT(
7660 if (Verbose && PrintGC) {
7661 gclog_or_tty->print("Collected "SIZE_FORMAT" objects, "
7662 SIZE_FORMAT " bytes",
7663 _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
7664 gclog_or_tty->print_cr("\nLive "SIZE_FORMAT" objects, "
7665 SIZE_FORMAT" bytes "
7666 "Already free "SIZE_FORMAT" objects, "SIZE_FORMAT" bytes",
7667 _numObjectsLive, _numWordsLive*sizeof(HeapWord),
7668 _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
7669 size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree) *
7670 sizeof(HeapWord);
7671 gclog_or_tty->print_cr("Total sweep: "SIZE_FORMAT" bytes", totalBytes);
7673 if (PrintCMSStatistics && CMSVerifyReturnedBytes) {
7674 size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
7675 size_t dictReturnedBytes = _sp->dictionary()->sumDictReturnedBytes();
7676 size_t returnedBytes = indexListReturnedBytes + dictReturnedBytes;
7677 gclog_or_tty->print("Returned "SIZE_FORMAT" bytes", returnedBytes);
7678 gclog_or_tty->print(" Indexed List Returned "SIZE_FORMAT" bytes",
7679 indexListReturnedBytes);
7680 gclog_or_tty->print_cr(" Dictionary Returned "SIZE_FORMAT" bytes",
7681 dictReturnedBytes);
7682 }
7683 }
7684 )
7685 // Now, in debug mode, just null out the sweep_limit
7686 NOT_PRODUCT(_sp->clear_sweep_limit();)
7687 if (CMSTraceSweeper) {
7688 gclog_or_tty->print("end of sweep\n================\n");
7689 }
7690 }
7692 void SweepClosure::initialize_free_range(HeapWord* freeFinger,
7693 bool freeRangeInFreeLists) {
7694 if (CMSTraceSweeper) {
7695 gclog_or_tty->print("---- Start free range 0x%x with free block [%d] (%d)\n",
7696 freeFinger, _sp->block_size(freeFinger),
7697 freeRangeInFreeLists);
7698 }
7699 assert(!inFreeRange(), "Trampling existing free range");
7700 set_inFreeRange(true);
7701 set_lastFreeRangeCoalesced(false);
7703 set_freeFinger(freeFinger);
7704 set_freeRangeInFreeLists(freeRangeInFreeLists);
7705 if (CMSTestInFreeList) {
7706 if (freeRangeInFreeLists) {
7707 FreeChunk* fc = (FreeChunk*) freeFinger;
7708 assert(fc->isFree(), "A chunk on the free list should be free.");
7709 assert(fc->size() > 0, "Free range should have a size");
7710 assert(_sp->verifyChunkInFreeLists(fc), "Chunk is not in free lists");
7711 }
7712 }
7713 }
7715 // Note that the sweeper runs concurrently with mutators. Thus,
7716 // it is possible for direct allocation in this generation to happen
7717 // in the middle of the sweep. Note that the sweeper also coalesces
7718 // contiguous free blocks. Thus, unless the sweeper and the allocator
7719 // synchronize appropriately freshly allocated blocks may get swept up.
7720 // This is accomplished by the sweeper locking the free lists while
7721 // it is sweeping. Thus blocks that are determined to be free are
7722 // indeed free. There is however one additional complication:
7723 // blocks that have been allocated since the final checkpoint and
7724 // mark, will not have been marked and so would be treated as
7725 // unreachable and swept up. To prevent this, the allocator marks
7726 // the bit map when allocating during the sweep phase. This leads,
7727 // however, to a further complication -- objects may have been allocated
7728 // but not yet initialized -- in the sense that the header isn't yet
7729 // installed. The sweeper can not then determine the size of the block
7730 // in order to skip over it. To deal with this case, we use a technique
7731 // (due to Printezis) to encode such uninitialized block sizes in the
7732 // bit map. Since the bit map uses a bit per every HeapWord, but the
7733 // CMS generation has a minimum object size of 3 HeapWords, it follows
7734 // that "normal marks" won't be adjacent in the bit map (there will
7735 // always be at least two 0 bits between successive 1 bits). We make use
7736 // of these "unused" bits to represent uninitialized blocks -- the bit
7737 // corresponding to the start of the uninitialized object and the next
7738 // bit are both set. Finally, a 1 bit marks the end of the object that
7739 // started with the two consecutive 1 bits to indicate its potentially
7740 // uninitialized state.
7742 size_t SweepClosure::do_blk_careful(HeapWord* addr) {
7743 FreeChunk* fc = (FreeChunk*)addr;
7744 size_t res;
7746 // check if we are done sweepinrg
7747 if (addr == _limit) { // we have swept up to the limit, do nothing more
7748 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7749 "sweep _limit out of bounds");
7750 // help the closure application finish
7751 return pointer_delta(_sp->end(), _limit);
7752 }
7753 assert(addr <= _limit, "sweep invariant");
7755 // check if we should yield
7756 do_yield_check(addr);
7757 if (fc->isFree()) {
7758 // Chunk that is already free
7759 res = fc->size();
7760 doAlreadyFreeChunk(fc);
7761 debug_only(_sp->verifyFreeLists());
7762 assert(res == fc->size(), "Don't expect the size to change");
7763 NOT_PRODUCT(
7764 _numObjectsAlreadyFree++;
7765 _numWordsAlreadyFree += res;
7766 )
7767 NOT_PRODUCT(_last_fc = fc;)
7768 } else if (!_bitMap->isMarked(addr)) {
7769 // Chunk is fresh garbage
7770 res = doGarbageChunk(fc);
7771 debug_only(_sp->verifyFreeLists());
7772 NOT_PRODUCT(
7773 _numObjectsFreed++;
7774 _numWordsFreed += res;
7775 )
7776 } else {
7777 // Chunk that is alive.
7778 res = doLiveChunk(fc);
7779 debug_only(_sp->verifyFreeLists());
7780 NOT_PRODUCT(
7781 _numObjectsLive++;
7782 _numWordsLive += res;
7783 )
7784 }
7785 return res;
7786 }
7788 // For the smart allocation, record following
7789 // split deaths - a free chunk is removed from its free list because
7790 // it is being split into two or more chunks.
7791 // split birth - a free chunk is being added to its free list because
7792 // a larger free chunk has been split and resulted in this free chunk.
7793 // coal death - a free chunk is being removed from its free list because
7794 // it is being coalesced into a large free chunk.
7795 // coal birth - a free chunk is being added to its free list because
7796 // it was created when two or more free chunks where coalesced into
7797 // this free chunk.
7798 //
7799 // These statistics are used to determine the desired number of free
7800 // chunks of a given size. The desired number is chosen to be relative
7801 // to the end of a CMS sweep. The desired number at the end of a sweep
7802 // is the
7803 // count-at-end-of-previous-sweep (an amount that was enough)
7804 // - count-at-beginning-of-current-sweep (the excess)
7805 // + split-births (gains in this size during interval)
7806 // - split-deaths (demands on this size during interval)
7807 // where the interval is from the end of one sweep to the end of the
7808 // next.
7809 //
7810 // When sweeping the sweeper maintains an accumulated chunk which is
7811 // the chunk that is made up of chunks that have been coalesced. That
7812 // will be termed the left-hand chunk. A new chunk of garbage that
7813 // is being considered for coalescing will be referred to as the
7814 // right-hand chunk.
7815 //
7816 // When making a decision on whether to coalesce a right-hand chunk with
7817 // the current left-hand chunk, the current count vs. the desired count
7818 // of the left-hand chunk is considered. Also if the right-hand chunk
7819 // is near the large chunk at the end of the heap (see
7820 // ConcurrentMarkSweepGeneration::isNearLargestChunk()), then the
7821 // left-hand chunk is coalesced.
7822 //
7823 // When making a decision about whether to split a chunk, the desired count
7824 // vs. the current count of the candidate to be split is also considered.
7825 // If the candidate is underpopulated (currently fewer chunks than desired)
7826 // a chunk of an overpopulated (currently more chunks than desired) size may
7827 // be chosen. The "hint" associated with a free list, if non-null, points
7828 // to a free list which may be overpopulated.
7829 //
7831 void SweepClosure::doAlreadyFreeChunk(FreeChunk* fc) {
7832 size_t size = fc->size();
7833 // Chunks that cannot be coalesced are not in the
7834 // free lists.
7835 if (CMSTestInFreeList && !fc->cantCoalesce()) {
7836 assert(_sp->verifyChunkInFreeLists(fc),
7837 "free chunk should be in free lists");
7838 }
7839 // a chunk that is already free, should not have been
7840 // marked in the bit map
7841 HeapWord* addr = (HeapWord*) fc;
7842 assert(!_bitMap->isMarked(addr), "free chunk should be unmarked");
7843 // Verify that the bit map has no bits marked between
7844 // addr and purported end of this block.
7845 _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
7847 // Some chunks cannot be coalesced in under any circumstances.
7848 // See the definition of cantCoalesce().
7849 if (!fc->cantCoalesce()) {
7850 // This chunk can potentially be coalesced.
7851 if (_sp->adaptive_freelists()) {
7852 // All the work is done in
7853 doPostIsFreeOrGarbageChunk(fc, size);
7854 } else { // Not adaptive free lists
7855 // this is a free chunk that can potentially be coalesced by the sweeper;
7856 if (!inFreeRange()) {
7857 // if the next chunk is a free block that can't be coalesced
7858 // it doesn't make sense to remove this chunk from the free lists
7859 FreeChunk* nextChunk = (FreeChunk*)(addr + size);
7860 assert((HeapWord*)nextChunk <= _limit, "sweep invariant");
7861 if ((HeapWord*)nextChunk < _limit && // there's a next chunk...
7862 nextChunk->isFree() && // which is free...
7863 nextChunk->cantCoalesce()) { // ... but cant be coalesced
7864 // nothing to do
7865 } else {
7866 // Potentially the start of a new free range:
7867 // Don't eagerly remove it from the free lists.
7868 // No need to remove it if it will just be put
7869 // back again. (Also from a pragmatic point of view
7870 // if it is a free block in a region that is beyond
7871 // any allocated blocks, an assertion will fail)
7872 // Remember the start of a free run.
7873 initialize_free_range(addr, true);
7874 // end - can coalesce with next chunk
7875 }
7876 } else {
7877 // the midst of a free range, we are coalescing
7878 debug_only(record_free_block_coalesced(fc);)
7879 if (CMSTraceSweeper) {
7880 gclog_or_tty->print(" -- pick up free block 0x%x (%d)\n", fc, size);
7881 }
7882 // remove it from the free lists
7883 _sp->removeFreeChunkFromFreeLists(fc);
7884 set_lastFreeRangeCoalesced(true);
7885 // If the chunk is being coalesced and the current free range is
7886 // in the free lists, remove the current free range so that it
7887 // will be returned to the free lists in its entirety - all
7888 // the coalesced pieces included.
7889 if (freeRangeInFreeLists()) {
7890 FreeChunk* ffc = (FreeChunk*) freeFinger();
7891 assert(ffc->size() == pointer_delta(addr, freeFinger()),
7892 "Size of free range is inconsistent with chunk size.");
7893 if (CMSTestInFreeList) {
7894 assert(_sp->verifyChunkInFreeLists(ffc),
7895 "free range is not in free lists");
7896 }
7897 _sp->removeFreeChunkFromFreeLists(ffc);
7898 set_freeRangeInFreeLists(false);
7899 }
7900 }
7901 }
7902 } else {
7903 // Code path common to both original and adaptive free lists.
7905 // cant coalesce with previous block; this should be treated
7906 // as the end of a free run if any
7907 if (inFreeRange()) {
7908 // we kicked some butt; time to pick up the garbage
7909 assert(freeFinger() < addr, "the finger pointeth off base");
7910 flushCurFreeChunk(freeFinger(), pointer_delta(addr, freeFinger()));
7911 }
7912 // else, nothing to do, just continue
7913 }
7914 }
7916 size_t SweepClosure::doGarbageChunk(FreeChunk* fc) {
7917 // This is a chunk of garbage. It is not in any free list.
7918 // Add it to a free list or let it possibly be coalesced into
7919 // a larger chunk.
7920 HeapWord* addr = (HeapWord*) fc;
7921 size_t size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
7923 if (_sp->adaptive_freelists()) {
7924 // Verify that the bit map has no bits marked between
7925 // addr and purported end of just dead object.
7926 _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
7928 doPostIsFreeOrGarbageChunk(fc, size);
7929 } else {
7930 if (!inFreeRange()) {
7931 // start of a new free range
7932 assert(size > 0, "A free range should have a size");
7933 initialize_free_range(addr, false);
7935 } else {
7936 // this will be swept up when we hit the end of the
7937 // free range
7938 if (CMSTraceSweeper) {
7939 gclog_or_tty->print(" -- pick up garbage 0x%x (%d) \n", fc, size);
7940 }
7941 // If the chunk is being coalesced and the current free range is
7942 // in the free lists, remove the current free range so that it
7943 // will be returned to the free lists in its entirety - all
7944 // the coalesced pieces included.
7945 if (freeRangeInFreeLists()) {
7946 FreeChunk* ffc = (FreeChunk*)freeFinger();
7947 assert(ffc->size() == pointer_delta(addr, freeFinger()),
7948 "Size of free range is inconsistent with chunk size.");
7949 if (CMSTestInFreeList) {
7950 assert(_sp->verifyChunkInFreeLists(ffc),
7951 "free range is not in free lists");
7952 }
7953 _sp->removeFreeChunkFromFreeLists(ffc);
7954 set_freeRangeInFreeLists(false);
7955 }
7956 set_lastFreeRangeCoalesced(true);
7957 }
7958 // this will be swept up when we hit the end of the free range
7960 // Verify that the bit map has no bits marked between
7961 // addr and purported end of just dead object.
7962 _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
7963 }
7964 return size;
7965 }
7967 size_t SweepClosure::doLiveChunk(FreeChunk* fc) {
7968 HeapWord* addr = (HeapWord*) fc;
7969 // The sweeper has just found a live object. Return any accumulated
7970 // left hand chunk to the free lists.
7971 if (inFreeRange()) {
7972 if (_sp->adaptive_freelists()) {
7973 flushCurFreeChunk(freeFinger(),
7974 pointer_delta(addr, freeFinger()));
7975 } else { // not adaptive freelists
7976 set_inFreeRange(false);
7977 // Add the free range back to the free list if it is not already
7978 // there.
7979 if (!freeRangeInFreeLists()) {
7980 assert(freeFinger() < addr, "the finger pointeth off base");
7981 if (CMSTraceSweeper) {
7982 gclog_or_tty->print("Sweep:put_free_blk 0x%x (%d) "
7983 "[coalesced:%d]\n",
7984 freeFinger(), pointer_delta(addr, freeFinger()),
7985 lastFreeRangeCoalesced());
7986 }
7987 _sp->addChunkAndRepairOffsetTable(freeFinger(),
7988 pointer_delta(addr, freeFinger()), lastFreeRangeCoalesced());
7989 }
7990 }
7991 }
7993 // Common code path for original and adaptive free lists.
7995 // this object is live: we'd normally expect this to be
7996 // an oop, and like to assert the following:
7997 // assert(oop(addr)->is_oop(), "live block should be an oop");
7998 // However, as we commented above, this may be an object whose
7999 // header hasn't yet been initialized.
8000 size_t size;
8001 assert(_bitMap->isMarked(addr), "Tautology for this control point");
8002 if (_bitMap->isMarked(addr + 1)) {
8003 // Determine the size from the bit map, rather than trying to
8004 // compute it from the object header.
8005 HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
8006 size = pointer_delta(nextOneAddr + 1, addr);
8007 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
8008 "alignment problem");
8010 #ifdef DEBUG
8011 if (oop(addr)->klass() != NULL &&
8012 ( !_collector->should_unload_classes()
8013 || oop(addr)->is_parsable())) {
8014 // Ignore mark word because we are running concurrent with mutators
8015 assert(oop(addr)->is_oop(true), "live block should be an oop");
8016 assert(size ==
8017 CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()),
8018 "P-mark and computed size do not agree");
8019 }
8020 #endif
8022 } else {
8023 // This should be an initialized object that's alive.
8024 assert(oop(addr)->klass() != NULL &&
8025 (!_collector->should_unload_classes()
8026 || oop(addr)->is_parsable()),
8027 "Should be an initialized object");
8028 // Ignore mark word because we are running concurrent with mutators
8029 assert(oop(addr)->is_oop(true), "live block should be an oop");
8030 // Verify that the bit map has no bits marked between
8031 // addr and purported end of this block.
8032 size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
8033 assert(size >= 3, "Necessary for Printezis marks to work");
8034 assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
8035 DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
8036 }
8037 return size;
8038 }
8040 void SweepClosure::doPostIsFreeOrGarbageChunk(FreeChunk* fc,
8041 size_t chunkSize) {
8042 // doPostIsFreeOrGarbageChunk() should only be called in the smart allocation
8043 // scheme.
8044 bool fcInFreeLists = fc->isFree();
8045 assert(_sp->adaptive_freelists(), "Should only be used in this case.");
8046 assert((HeapWord*)fc <= _limit, "sweep invariant");
8047 if (CMSTestInFreeList && fcInFreeLists) {
8048 assert(_sp->verifyChunkInFreeLists(fc),
8049 "free chunk is not in free lists");
8050 }
8053 if (CMSTraceSweeper) {
8054 gclog_or_tty->print_cr(" -- pick up another chunk at 0x%x (%d)", fc, chunkSize);
8055 }
8057 HeapWord* addr = (HeapWord*) fc;
8059 bool coalesce;
8060 size_t left = pointer_delta(addr, freeFinger());
8061 size_t right = chunkSize;
8062 switch (FLSCoalescePolicy) {
8063 // numeric value forms a coalition aggressiveness metric
8064 case 0: { // never coalesce
8065 coalesce = false;
8066 break;
8067 }
8068 case 1: { // coalesce if left & right chunks on overpopulated lists
8069 coalesce = _sp->coalOverPopulated(left) &&
8070 _sp->coalOverPopulated(right);
8071 break;
8072 }
8073 case 2: { // coalesce if left chunk on overpopulated list (default)
8074 coalesce = _sp->coalOverPopulated(left);
8075 break;
8076 }
8077 case 3: { // coalesce if left OR right chunk on overpopulated list
8078 coalesce = _sp->coalOverPopulated(left) ||
8079 _sp->coalOverPopulated(right);
8080 break;
8081 }
8082 case 4: { // always coalesce
8083 coalesce = true;
8084 break;
8085 }
8086 default:
8087 ShouldNotReachHere();
8088 }
8090 // Should the current free range be coalesced?
8091 // If the chunk is in a free range and either we decided to coalesce above
8092 // or the chunk is near the large block at the end of the heap
8093 // (isNearLargestChunk() returns true), then coalesce this chunk.
8094 bool doCoalesce = inFreeRange() &&
8095 (coalesce || _g->isNearLargestChunk((HeapWord*)fc));
8096 if (doCoalesce) {
8097 // Coalesce the current free range on the left with the new
8098 // chunk on the right. If either is on a free list,
8099 // it must be removed from the list and stashed in the closure.
8100 if (freeRangeInFreeLists()) {
8101 FreeChunk* ffc = (FreeChunk*)freeFinger();
8102 assert(ffc->size() == pointer_delta(addr, freeFinger()),
8103 "Size of free range is inconsistent with chunk size.");
8104 if (CMSTestInFreeList) {
8105 assert(_sp->verifyChunkInFreeLists(ffc),
8106 "Chunk is not in free lists");
8107 }
8108 _sp->coalDeath(ffc->size());
8109 _sp->removeFreeChunkFromFreeLists(ffc);
8110 set_freeRangeInFreeLists(false);
8111 }
8112 if (fcInFreeLists) {
8113 _sp->coalDeath(chunkSize);
8114 assert(fc->size() == chunkSize,
8115 "The chunk has the wrong size or is not in the free lists");
8116 _sp->removeFreeChunkFromFreeLists(fc);
8117 }
8118 set_lastFreeRangeCoalesced(true);
8119 } else { // not in a free range and/or should not coalesce
8120 // Return the current free range and start a new one.
8121 if (inFreeRange()) {
8122 // In a free range but cannot coalesce with the right hand chunk.
8123 // Put the current free range into the free lists.
8124 flushCurFreeChunk(freeFinger(),
8125 pointer_delta(addr, freeFinger()));
8126 }
8127 // Set up for new free range. Pass along whether the right hand
8128 // chunk is in the free lists.
8129 initialize_free_range((HeapWord*)fc, fcInFreeLists);
8130 }
8131 }
8132 void SweepClosure::flushCurFreeChunk(HeapWord* chunk, size_t size) {
8133 assert(inFreeRange(), "Should only be called if currently in a free range.");
8134 assert(size > 0,
8135 "A zero sized chunk cannot be added to the free lists.");
8136 if (!freeRangeInFreeLists()) {
8137 if(CMSTestInFreeList) {
8138 FreeChunk* fc = (FreeChunk*) chunk;
8139 fc->setSize(size);
8140 assert(!_sp->verifyChunkInFreeLists(fc),
8141 "chunk should not be in free lists yet");
8142 }
8143 if (CMSTraceSweeper) {
8144 gclog_or_tty->print_cr(" -- add free block 0x%x (%d) to free lists",
8145 chunk, size);
8146 }
8147 // A new free range is going to be starting. The current
8148 // free range has not been added to the free lists yet or
8149 // was removed so add it back.
8150 // If the current free range was coalesced, then the death
8151 // of the free range was recorded. Record a birth now.
8152 if (lastFreeRangeCoalesced()) {
8153 _sp->coalBirth(size);
8154 }
8155 _sp->addChunkAndRepairOffsetTable(chunk, size,
8156 lastFreeRangeCoalesced());
8157 }
8158 set_inFreeRange(false);
8159 set_freeRangeInFreeLists(false);
8160 }
8162 // We take a break if we've been at this for a while,
8163 // so as to avoid monopolizing the locks involved.
8164 void SweepClosure::do_yield_work(HeapWord* addr) {
8165 // Return current free chunk being used for coalescing (if any)
8166 // to the appropriate freelist. After yielding, the next
8167 // free block encountered will start a coalescing range of
8168 // free blocks. If the next free block is adjacent to the
8169 // chunk just flushed, they will need to wait for the next
8170 // sweep to be coalesced.
8171 if (inFreeRange()) {
8172 flushCurFreeChunk(freeFinger(), pointer_delta(addr, freeFinger()));
8173 }
8175 // First give up the locks, then yield, then re-lock.
8176 // We should probably use a constructor/destructor idiom to
8177 // do this unlock/lock or modify the MutexUnlocker class to
8178 // serve our purpose. XXX
8179 assert_lock_strong(_bitMap->lock());
8180 assert_lock_strong(_freelistLock);
8181 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
8182 "CMS thread should hold CMS token");
8183 _bitMap->lock()->unlock();
8184 _freelistLock->unlock();
8185 ConcurrentMarkSweepThread::desynchronize(true);
8186 ConcurrentMarkSweepThread::acknowledge_yield_request();
8187 _collector->stopTimer();
8188 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
8189 if (PrintCMSStatistics != 0) {
8190 _collector->incrementYields();
8191 }
8192 _collector->icms_wait();
8194 // See the comment in coordinator_yield()
8195 for (unsigned i = 0; i < CMSYieldSleepCount &&
8196 ConcurrentMarkSweepThread::should_yield() &&
8197 !CMSCollector::foregroundGCIsActive(); ++i) {
8198 os::sleep(Thread::current(), 1, false);
8199 ConcurrentMarkSweepThread::acknowledge_yield_request();
8200 }
8202 ConcurrentMarkSweepThread::synchronize(true);
8203 _freelistLock->lock();
8204 _bitMap->lock()->lock_without_safepoint_check();
8205 _collector->startTimer();
8206 }
8208 #ifndef PRODUCT
8209 // This is actually very useful in a product build if it can
8210 // be called from the debugger. Compile it into the product
8211 // as needed.
8212 bool debug_verifyChunkInFreeLists(FreeChunk* fc) {
8213 return debug_cms_space->verifyChunkInFreeLists(fc);
8214 }
8216 void SweepClosure::record_free_block_coalesced(FreeChunk* fc) const {
8217 if (CMSTraceSweeper) {
8218 gclog_or_tty->print("Sweep:coal_free_blk 0x%x (%d)\n", fc, fc->size());
8219 }
8220 }
8221 #endif
8223 // CMSIsAliveClosure
8224 bool CMSIsAliveClosure::do_object_b(oop obj) {
8225 HeapWord* addr = (HeapWord*)obj;
8226 return addr != NULL &&
8227 (!_span.contains(addr) || _bit_map->isMarked(addr));
8228 }
8230 // CMSKeepAliveClosure: the serial version
8231 void CMSKeepAliveClosure::do_oop(oop* p) {
8232 oop this_oop = *p;
8233 HeapWord* addr = (HeapWord*)this_oop;
8234 if (_span.contains(addr) &&
8235 !_bit_map->isMarked(addr)) {
8236 _bit_map->mark(addr);
8237 bool simulate_overflow = false;
8238 NOT_PRODUCT(
8239 if (CMSMarkStackOverflowALot &&
8240 _collector->simulate_overflow()) {
8241 // simulate a stack overflow
8242 simulate_overflow = true;
8243 }
8244 )
8245 if (simulate_overflow || !_mark_stack->push(this_oop)) {
8246 _collector->push_on_overflow_list(this_oop);
8247 _collector->_ser_kac_ovflw++;
8248 }
8249 }
8250 }
8252 // CMSParKeepAliveClosure: a parallel version of the above.
8253 // The work queues are private to each closure (thread),
8254 // but (may be) available for stealing by other threads.
8255 void CMSParKeepAliveClosure::do_oop(oop* p) {
8256 oop this_oop = *p;
8257 HeapWord* addr = (HeapWord*)this_oop;
8258 if (_span.contains(addr) &&
8259 !_bit_map->isMarked(addr)) {
8260 // In general, during recursive tracing, several threads
8261 // may be concurrently getting here; the first one to
8262 // "tag" it, claims it.
8263 if (_bit_map->par_mark(addr)) {
8264 bool res = _work_queue->push(this_oop);
8265 assert(res, "Low water mark should be much less than capacity");
8266 // Do a recursive trim in the hope that this will keep
8267 // stack usage lower, but leave some oops for potential stealers
8268 trim_queue(_low_water_mark);
8269 } // Else, another thread got there first
8270 }
8271 }
8273 void CMSParKeepAliveClosure::trim_queue(uint max) {
8274 while (_work_queue->size() > max) {
8275 oop new_oop;
8276 if (_work_queue->pop_local(new_oop)) {
8277 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
8278 assert(_bit_map->isMarked((HeapWord*)new_oop),
8279 "no white objects on this stack!");
8280 assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop");
8281 // iterate over the oops in this oop, marking and pushing
8282 // the ones in CMS heap (i.e. in _span).
8283 new_oop->oop_iterate(&_mark_and_push);
8284 }
8285 }
8286 }
8288 void CMSInnerParMarkAndPushClosure::do_oop(oop* p) {
8289 oop this_oop = *p;
8290 HeapWord* addr = (HeapWord*)this_oop;
8291 if (_span.contains(addr) &&
8292 !_bit_map->isMarked(addr)) {
8293 if (_bit_map->par_mark(addr)) {
8294 bool simulate_overflow = false;
8295 NOT_PRODUCT(
8296 if (CMSMarkStackOverflowALot &&
8297 _collector->par_simulate_overflow()) {
8298 // simulate a stack overflow
8299 simulate_overflow = true;
8300 }
8301 )
8302 if (simulate_overflow || !_work_queue->push(this_oop)) {
8303 _collector->par_push_on_overflow_list(this_oop);
8304 _collector->_par_kac_ovflw++;
8305 }
8306 } // Else another thread got there already
8307 }
8308 }
8310 //////////////////////////////////////////////////////////////////
8311 // CMSExpansionCause /////////////////////////////
8312 //////////////////////////////////////////////////////////////////
8313 const char* CMSExpansionCause::to_string(CMSExpansionCause::Cause cause) {
8314 switch (cause) {
8315 case _no_expansion:
8316 return "No expansion";
8317 case _satisfy_free_ratio:
8318 return "Free ratio";
8319 case _satisfy_promotion:
8320 return "Satisfy promotion";
8321 case _satisfy_allocation:
8322 return "allocation";
8323 case _allocate_par_lab:
8324 return "Par LAB";
8325 case _allocate_par_spooling_space:
8326 return "Par Spooling Space";
8327 case _adaptive_size_policy:
8328 return "Ergonomics";
8329 default:
8330 return "unknown";
8331 }
8332 }
8334 void CMSDrainMarkingStackClosure::do_void() {
8335 // the max number to take from overflow list at a time
8336 const size_t num = _mark_stack->capacity()/4;
8337 while (!_mark_stack->isEmpty() ||
8338 // if stack is empty, check the overflow list
8339 _collector->take_from_overflow_list(num, _mark_stack)) {
8340 oop this_oop = _mark_stack->pop();
8341 HeapWord* addr = (HeapWord*)this_oop;
8342 assert(_span.contains(addr), "Should be within span");
8343 assert(_bit_map->isMarked(addr), "Should be marked");
8344 assert(this_oop->is_oop(), "Should be an oop");
8345 this_oop->oop_iterate(_keep_alive);
8346 }
8347 }
8349 void CMSParDrainMarkingStackClosure::do_void() {
8350 // drain queue
8351 trim_queue(0);
8352 }
8354 // Trim our work_queue so its length is below max at return
8355 void CMSParDrainMarkingStackClosure::trim_queue(uint max) {
8356 while (_work_queue->size() > max) {
8357 oop new_oop;
8358 if (_work_queue->pop_local(new_oop)) {
8359 assert(new_oop->is_oop(), "Expected an oop");
8360 assert(_bit_map->isMarked((HeapWord*)new_oop),
8361 "no white objects on this stack!");
8362 assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop");
8363 // iterate over the oops in this oop, marking and pushing
8364 // the ones in CMS heap (i.e. in _span).
8365 new_oop->oop_iterate(&_mark_and_push);
8366 }
8367 }
8368 }
8370 ////////////////////////////////////////////////////////////////////
8371 // Support for Marking Stack Overflow list handling and related code
8372 ////////////////////////////////////////////////////////////////////
8373 // Much of the following code is similar in shape and spirit to the
8374 // code used in ParNewGC. We should try and share that code
8375 // as much as possible in the future.
8377 #ifndef PRODUCT
8378 // Debugging support for CMSStackOverflowALot
8380 // It's OK to call this multi-threaded; the worst thing
8381 // that can happen is that we'll get a bunch of closely
8382 // spaced simulated oveflows, but that's OK, in fact
8383 // probably good as it would exercise the overflow code
8384 // under contention.
8385 bool CMSCollector::simulate_overflow() {
8386 if (_overflow_counter-- <= 0) { // just being defensive
8387 _overflow_counter = CMSMarkStackOverflowInterval;
8388 return true;
8389 } else {
8390 return false;
8391 }
8392 }
8394 bool CMSCollector::par_simulate_overflow() {
8395 return simulate_overflow();
8396 }
8397 #endif
8399 // Single-threaded
8400 bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) {
8401 assert(stack->isEmpty(), "Expected precondition");
8402 assert(stack->capacity() > num, "Shouldn't bite more than can chew");
8403 size_t i = num;
8404 oop cur = _overflow_list;
8405 const markOop proto = markOopDesc::prototype();
8406 NOT_PRODUCT(size_t n = 0;)
8407 for (oop next; i > 0 && cur != NULL; cur = next, i--) {
8408 next = oop(cur->mark());
8409 cur->set_mark(proto); // until proven otherwise
8410 assert(cur->is_oop(), "Should be an oop");
8411 bool res = stack->push(cur);
8412 assert(res, "Bit off more than can chew?");
8413 NOT_PRODUCT(n++;)
8414 }
8415 _overflow_list = cur;
8416 #ifndef PRODUCT
8417 assert(_num_par_pushes >= n, "Too many pops?");
8418 _num_par_pushes -=n;
8419 #endif
8420 return !stack->isEmpty();
8421 }
8423 // Multi-threaded; use CAS to break off a prefix
8424 bool CMSCollector::par_take_from_overflow_list(size_t num,
8425 OopTaskQueue* work_q) {
8426 assert(work_q->size() == 0, "That's the current policy");
8427 assert(num < work_q->max_elems(), "Can't bite more than we can chew");
8428 if (_overflow_list == NULL) {
8429 return false;
8430 }
8431 // Grab the entire list; we'll put back a suffix
8432 oop prefix = (oop)Atomic::xchg_ptr(NULL, &_overflow_list);
8433 if (prefix == NULL) { // someone grabbed it before we did ...
8434 // ... we could spin for a short while, but for now we don't
8435 return false;
8436 }
8437 size_t i = num;
8438 oop cur = prefix;
8439 for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--);
8440 if (cur->mark() != NULL) {
8441 oop suffix_head = cur->mark(); // suffix will be put back on global list
8442 cur->set_mark(NULL); // break off suffix
8443 // Find tail of suffix so we can prepend suffix to global list
8444 for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark()));
8445 oop suffix_tail = cur;
8446 assert(suffix_tail != NULL && suffix_tail->mark() == NULL,
8447 "Tautology");
8448 oop observed_overflow_list = _overflow_list;
8449 do {
8450 cur = observed_overflow_list;
8451 suffix_tail->set_mark(markOop(cur));
8452 observed_overflow_list =
8453 (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur);
8454 } while (cur != observed_overflow_list);
8455 }
8457 // Push the prefix elements on work_q
8458 assert(prefix != NULL, "control point invariant");
8459 const markOop proto = markOopDesc::prototype();
8460 oop next;
8461 NOT_PRODUCT(size_t n = 0;)
8462 for (cur = prefix; cur != NULL; cur = next) {
8463 next = oop(cur->mark());
8464 cur->set_mark(proto); // until proven otherwise
8465 assert(cur->is_oop(), "Should be an oop");
8466 bool res = work_q->push(cur);
8467 assert(res, "Bit off more than we can chew?");
8468 NOT_PRODUCT(n++;)
8469 }
8470 #ifndef PRODUCT
8471 assert(_num_par_pushes >= n, "Too many pops?");
8472 Atomic::add_ptr(-(intptr_t)n, &_num_par_pushes);
8473 #endif
8474 return true;
8475 }
8477 // Single-threaded
8478 void CMSCollector::push_on_overflow_list(oop p) {
8479 NOT_PRODUCT(_num_par_pushes++;)
8480 assert(p->is_oop(), "Not an oop");
8481 preserve_mark_if_necessary(p);
8482 p->set_mark((markOop)_overflow_list);
8483 _overflow_list = p;
8484 }
8486 // Multi-threaded; use CAS to prepend to overflow list
8487 void CMSCollector::par_push_on_overflow_list(oop p) {
8488 NOT_PRODUCT(Atomic::inc_ptr(&_num_par_pushes);)
8489 assert(p->is_oop(), "Not an oop");
8490 par_preserve_mark_if_necessary(p);
8491 oop observed_overflow_list = _overflow_list;
8492 oop cur_overflow_list;
8493 do {
8494 cur_overflow_list = observed_overflow_list;
8495 p->set_mark(markOop(cur_overflow_list));
8496 observed_overflow_list =
8497 (oop) Atomic::cmpxchg_ptr(p, &_overflow_list, cur_overflow_list);
8498 } while (cur_overflow_list != observed_overflow_list);
8499 }
8501 // Single threaded
8502 // General Note on GrowableArray: pushes may silently fail
8503 // because we are (temporarily) out of C-heap for expanding
8504 // the stack. The problem is quite ubiquitous and affects
8505 // a lot of code in the JVM. The prudent thing for GrowableArray
8506 // to do (for now) is to exit with an error. However, that may
8507 // be too draconian in some cases because the caller may be
8508 // able to recover without much harm. For suych cases, we
8509 // should probably introduce a "soft_push" method which returns
8510 // an indication of success or failure with the assumption that
8511 // the caller may be able to recover from a failure; code in
8512 // the VM can then be changed, incrementally, to deal with such
8513 // failures where possible, thus, incrementally hardening the VM
8514 // in such low resource situations.
8515 void CMSCollector::preserve_mark_work(oop p, markOop m) {
8516 int PreserveMarkStackSize = 128;
8518 if (_preserved_oop_stack == NULL) {
8519 assert(_preserved_mark_stack == NULL,
8520 "bijection with preserved_oop_stack");
8521 // Allocate the stacks
8522 _preserved_oop_stack = new (ResourceObj::C_HEAP)
8523 GrowableArray<oop>(PreserveMarkStackSize, true);
8524 _preserved_mark_stack = new (ResourceObj::C_HEAP)
8525 GrowableArray<markOop>(PreserveMarkStackSize, true);
8526 if (_preserved_oop_stack == NULL || _preserved_mark_stack == NULL) {
8527 vm_exit_out_of_memory(2* PreserveMarkStackSize * sizeof(oop) /* punt */,
8528 "Preserved Mark/Oop Stack for CMS (C-heap)");
8529 }
8530 }
8531 _preserved_oop_stack->push(p);
8532 _preserved_mark_stack->push(m);
8533 assert(m == p->mark(), "Mark word changed");
8534 assert(_preserved_oop_stack->length() == _preserved_mark_stack->length(),
8535 "bijection");
8536 }
8538 // Single threaded
8539 void CMSCollector::preserve_mark_if_necessary(oop p) {
8540 markOop m = p->mark();
8541 if (m->must_be_preserved(p)) {
8542 preserve_mark_work(p, m);
8543 }
8544 }
8546 void CMSCollector::par_preserve_mark_if_necessary(oop p) {
8547 markOop m = p->mark();
8548 if (m->must_be_preserved(p)) {
8549 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
8550 // Even though we read the mark word without holding
8551 // the lock, we are assured that it will not change
8552 // because we "own" this oop, so no other thread can
8553 // be trying to push it on the overflow list; see
8554 // the assertion in preserve_mark_work() that checks
8555 // that m == p->mark().
8556 preserve_mark_work(p, m);
8557 }
8558 }
8560 // We should be able to do this multi-threaded,
8561 // a chunk of stack being a task (this is
8562 // correct because each oop only ever appears
8563 // once in the overflow list. However, it's
8564 // not very easy to completely overlap this with
8565 // other operations, so will generally not be done
8566 // until all work's been completed. Because we
8567 // expect the preserved oop stack (set) to be small,
8568 // it's probably fine to do this single-threaded.
8569 // We can explore cleverer concurrent/overlapped/parallel
8570 // processing of preserved marks if we feel the
8571 // need for this in the future. Stack overflow should
8572 // be so rare in practice and, when it happens, its
8573 // effect on performance so great that this will
8574 // likely just be in the noise anyway.
8575 void CMSCollector::restore_preserved_marks_if_any() {
8576 if (_preserved_oop_stack == NULL) {
8577 assert(_preserved_mark_stack == NULL,
8578 "bijection with preserved_oop_stack");
8579 return;
8580 }
8582 assert(SafepointSynchronize::is_at_safepoint(),
8583 "world should be stopped");
8584 assert(Thread::current()->is_ConcurrentGC_thread() ||
8585 Thread::current()->is_VM_thread(),
8586 "should be single-threaded");
8588 int length = _preserved_oop_stack->length();
8589 assert(_preserved_mark_stack->length() == length, "bijection");
8590 for (int i = 0; i < length; i++) {
8591 oop p = _preserved_oop_stack->at(i);
8592 assert(p->is_oop(), "Should be an oop");
8593 assert(_span.contains(p), "oop should be in _span");
8594 assert(p->mark() == markOopDesc::prototype(),
8595 "Set when taken from overflow list");
8596 markOop m = _preserved_mark_stack->at(i);
8597 p->set_mark(m);
8598 }
8599 _preserved_mark_stack->clear();
8600 _preserved_oop_stack->clear();
8601 assert(_preserved_mark_stack->is_empty() &&
8602 _preserved_oop_stack->is_empty(),
8603 "stacks were cleared above");
8604 }
8606 #ifndef PRODUCT
8607 bool CMSCollector::no_preserved_marks() const {
8608 return ( ( _preserved_mark_stack == NULL
8609 && _preserved_oop_stack == NULL)
8610 || ( _preserved_mark_stack->is_empty()
8611 && _preserved_oop_stack->is_empty()));
8612 }
8613 #endif
8615 CMSAdaptiveSizePolicy* ASConcurrentMarkSweepGeneration::cms_size_policy() const
8616 {
8617 GenCollectedHeap* gch = (GenCollectedHeap*) GenCollectedHeap::heap();
8618 CMSAdaptiveSizePolicy* size_policy =
8619 (CMSAdaptiveSizePolicy*) gch->gen_policy()->size_policy();
8620 assert(size_policy->is_gc_cms_adaptive_size_policy(),
8621 "Wrong type for size policy");
8622 return size_policy;
8623 }
8625 void ASConcurrentMarkSweepGeneration::resize(size_t cur_promo_size,
8626 size_t desired_promo_size) {
8627 if (cur_promo_size < desired_promo_size) {
8628 size_t expand_bytes = desired_promo_size - cur_promo_size;
8629 if (PrintAdaptiveSizePolicy && Verbose) {
8630 gclog_or_tty->print_cr(" ASConcurrentMarkSweepGeneration::resize "
8631 "Expanding tenured generation by " SIZE_FORMAT " (bytes)",
8632 expand_bytes);
8633 }
8634 expand(expand_bytes,
8635 MinHeapDeltaBytes,
8636 CMSExpansionCause::_adaptive_size_policy);
8637 } else if (desired_promo_size < cur_promo_size) {
8638 size_t shrink_bytes = cur_promo_size - desired_promo_size;
8639 if (PrintAdaptiveSizePolicy && Verbose) {
8640 gclog_or_tty->print_cr(" ASConcurrentMarkSweepGeneration::resize "
8641 "Shrinking tenured generation by " SIZE_FORMAT " (bytes)",
8642 shrink_bytes);
8643 }
8644 shrink(shrink_bytes);
8645 }
8646 }
8648 CMSGCAdaptivePolicyCounters* ASConcurrentMarkSweepGeneration::gc_adaptive_policy_counters() {
8649 GenCollectedHeap* gch = GenCollectedHeap::heap();
8650 CMSGCAdaptivePolicyCounters* counters =
8651 (CMSGCAdaptivePolicyCounters*) gch->collector_policy()->counters();
8652 assert(counters->kind() == GCPolicyCounters::CMSGCAdaptivePolicyCountersKind,
8653 "Wrong kind of counters");
8654 return counters;
8655 }
8658 void ASConcurrentMarkSweepGeneration::update_counters() {
8659 if (UsePerfData) {
8660 _space_counters->update_all();
8661 _gen_counters->update_all();
8662 CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
8663 GenCollectedHeap* gch = GenCollectedHeap::heap();
8664 CMSGCStats* gc_stats_l = (CMSGCStats*) gc_stats();
8665 assert(gc_stats_l->kind() == GCStats::CMSGCStatsKind,
8666 "Wrong gc statistics type");
8667 counters->update_counters(gc_stats_l);
8668 }
8669 }
8671 void ASConcurrentMarkSweepGeneration::update_counters(size_t used) {
8672 if (UsePerfData) {
8673 _space_counters->update_used(used);
8674 _space_counters->update_capacity();
8675 _gen_counters->update_all();
8677 CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
8678 GenCollectedHeap* gch = GenCollectedHeap::heap();
8679 CMSGCStats* gc_stats_l = (CMSGCStats*) gc_stats();
8680 assert(gc_stats_l->kind() == GCStats::CMSGCStatsKind,
8681 "Wrong gc statistics type");
8682 counters->update_counters(gc_stats_l);
8683 }
8684 }
8686 // The desired expansion delta is computed so that:
8687 // . desired free percentage or greater is used
8688 void ASConcurrentMarkSweepGeneration::compute_new_size() {
8689 assert_locked_or_safepoint(Heap_lock);
8691 GenCollectedHeap* gch = (GenCollectedHeap*) GenCollectedHeap::heap();
8693 // If incremental collection failed, we just want to expand
8694 // to the limit.
8695 if (incremental_collection_failed()) {
8696 clear_incremental_collection_failed();
8697 grow_to_reserved();
8698 return;
8699 }
8701 assert(UseAdaptiveSizePolicy, "Should be using adaptive sizing");
8703 assert(gch->kind() == CollectedHeap::GenCollectedHeap,
8704 "Wrong type of heap");
8705 int prev_level = level() - 1;
8706 assert(prev_level >= 0, "The cms generation is the lowest generation");
8707 Generation* prev_gen = gch->get_gen(prev_level);
8708 assert(prev_gen->kind() == Generation::ASParNew,
8709 "Wrong type of young generation");
8710 ParNewGeneration* younger_gen = (ParNewGeneration*) prev_gen;
8711 size_t cur_eden = younger_gen->eden()->capacity();
8712 CMSAdaptiveSizePolicy* size_policy = cms_size_policy();
8713 size_t cur_promo = free();
8714 size_policy->compute_tenured_generation_free_space(cur_promo,
8715 max_available(),
8716 cur_eden);
8717 resize(cur_promo, size_policy->promo_size());
8719 // Record the new size of the space in the cms generation
8720 // that is available for promotions. This is temporary.
8721 // It should be the desired promo size.
8722 size_policy->avg_cms_promo()->sample(free());
8723 size_policy->avg_old_live()->sample(used());
8725 if (UsePerfData) {
8726 CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
8727 counters->update_cms_capacity_counter(capacity());
8728 }
8729 }
8731 void ASConcurrentMarkSweepGeneration::shrink_by(size_t desired_bytes) {
8732 assert_locked_or_safepoint(Heap_lock);
8733 assert_lock_strong(freelistLock());
8734 HeapWord* old_end = _cmsSpace->end();
8735 HeapWord* unallocated_start = _cmsSpace->unallocated_block();
8736 assert(old_end >= unallocated_start, "Miscalculation of unallocated_start");
8737 FreeChunk* chunk_at_end = find_chunk_at_end();
8738 if (chunk_at_end == NULL) {
8739 // No room to shrink
8740 if (PrintGCDetails && Verbose) {
8741 gclog_or_tty->print_cr("No room to shrink: old_end "
8742 PTR_FORMAT " unallocated_start " PTR_FORMAT
8743 " chunk_at_end " PTR_FORMAT,
8744 old_end, unallocated_start, chunk_at_end);
8745 }
8746 return;
8747 } else {
8749 // Find the chunk at the end of the space and determine
8750 // how much it can be shrunk.
8751 size_t shrinkable_size_in_bytes = chunk_at_end->size();
8752 size_t aligned_shrinkable_size_in_bytes =
8753 align_size_down(shrinkable_size_in_bytes, os::vm_page_size());
8754 assert(unallocated_start <= chunk_at_end->end(),
8755 "Inconsistent chunk at end of space");
8756 size_t bytes = MIN2(desired_bytes, aligned_shrinkable_size_in_bytes);
8757 size_t word_size_before = heap_word_size(_virtual_space.committed_size());
8759 // Shrink the underlying space
8760 _virtual_space.shrink_by(bytes);
8761 if (PrintGCDetails && Verbose) {
8762 gclog_or_tty->print_cr("ConcurrentMarkSweepGeneration::shrink_by:"
8763 " desired_bytes " SIZE_FORMAT
8764 " shrinkable_size_in_bytes " SIZE_FORMAT
8765 " aligned_shrinkable_size_in_bytes " SIZE_FORMAT
8766 " bytes " SIZE_FORMAT,
8767 desired_bytes, shrinkable_size_in_bytes,
8768 aligned_shrinkable_size_in_bytes, bytes);
8769 gclog_or_tty->print_cr(" old_end " SIZE_FORMAT
8770 " unallocated_start " SIZE_FORMAT,
8771 old_end, unallocated_start);
8772 }
8774 // If the space did shrink (shrinking is not guaranteed),
8775 // shrink the chunk at the end by the appropriate amount.
8776 if (((HeapWord*)_virtual_space.high()) < old_end) {
8777 size_t new_word_size =
8778 heap_word_size(_virtual_space.committed_size());
8780 // Have to remove the chunk from the dictionary because it is changing
8781 // size and might be someplace elsewhere in the dictionary.
8783 // Get the chunk at end, shrink it, and put it
8784 // back.
8785 _cmsSpace->removeChunkFromDictionary(chunk_at_end);
8786 size_t word_size_change = word_size_before - new_word_size;
8787 size_t chunk_at_end_old_size = chunk_at_end->size();
8788 assert(chunk_at_end_old_size >= word_size_change,
8789 "Shrink is too large");
8790 chunk_at_end->setSize(chunk_at_end_old_size -
8791 word_size_change);
8792 _cmsSpace->freed((HeapWord*) chunk_at_end->end(),
8793 word_size_change);
8795 _cmsSpace->returnChunkToDictionary(chunk_at_end);
8797 MemRegion mr(_cmsSpace->bottom(), new_word_size);
8798 _bts->resize(new_word_size); // resize the block offset shared array
8799 Universe::heap()->barrier_set()->resize_covered_region(mr);
8800 _cmsSpace->assert_locked();
8801 _cmsSpace->set_end((HeapWord*)_virtual_space.high());
8803 NOT_PRODUCT(_cmsSpace->dictionary()->verify());
8805 // update the space and generation capacity counters
8806 if (UsePerfData) {
8807 _space_counters->update_capacity();
8808 _gen_counters->update_all();
8809 }
8811 if (Verbose && PrintGCDetails) {
8812 size_t new_mem_size = _virtual_space.committed_size();
8813 size_t old_mem_size = new_mem_size + bytes;
8814 gclog_or_tty->print_cr("Shrinking %s from %ldK by %ldK to %ldK",
8815 name(), old_mem_size/K, bytes/K, new_mem_size/K);
8816 }
8817 }
8819 assert(_cmsSpace->unallocated_block() <= _cmsSpace->end(),
8820 "Inconsistency at end of space");
8821 assert(chunk_at_end->end() == _cmsSpace->end(),
8822 "Shrinking is inconsistent");
8823 return;
8824 }
8825 }
8827 // Transfer some number of overflown objects to usual marking
8828 // stack. Return true if some objects were transferred.
8829 bool MarkRefsIntoAndScanClosure::take_from_overflow_list() {
8830 size_t num = MIN2((size_t)_mark_stack->capacity()/4,
8831 (size_t)ParGCDesiredObjsFromOverflowList);
8833 bool res = _collector->take_from_overflow_list(num, _mark_stack);
8834 assert(_collector->overflow_list_is_empty() || res,
8835 "If list is not empty, we should have taken something");
8836 assert(!res || !_mark_stack->isEmpty(),
8837 "If we took something, it should now be on our stack");
8838 return res;
8839 }
8841 size_t MarkDeadObjectsClosure::do_blk(HeapWord* addr) {
8842 size_t res = _sp->block_size_no_stall(addr, _collector);
8843 assert(res != 0, "Should always be able to compute a size");
8844 if (_sp->block_is_obj(addr)) {
8845 if (_live_bit_map->isMarked(addr)) {
8846 // It can't have been dead in a previous cycle
8847 guarantee(!_dead_bit_map->isMarked(addr), "No resurrection!");
8848 } else {
8849 _dead_bit_map->mark(addr); // mark the dead object
8850 }
8851 }
8852 return res;
8853 }