Mon, 29 Jun 2009 12:17:03 -0400
6855115: G1: Fix for 6850869 is incorrect
Summary: Missed updating two variable names when improving the code for 6850869.
Reviewed-by: iveresov, jmasa, ysr
1 /*
2 * Copyright 2001-2008 Sun Microsystems, Inc. All Rights Reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
20 * CA 95054 USA or visit www.sun.com if you need additional information or
21 * have any questions.
22 *
23 */
25 #include "incls/_precompiled.incl"
26 #include "incls/_concurrentMark.cpp.incl"
28 //
29 // CMS Bit Map Wrapper
31 CMBitMapRO::CMBitMapRO(ReservedSpace rs, int shifter):
32 _bm((uintptr_t*)NULL,0),
33 _shifter(shifter) {
34 _bmStartWord = (HeapWord*)(rs.base());
35 _bmWordSize = rs.size()/HeapWordSize; // rs.size() is in bytes
36 ReservedSpace brs(ReservedSpace::allocation_align_size_up(
37 (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1));
39 guarantee(brs.is_reserved(), "couldn't allocate CMS bit map");
40 // For now we'll just commit all of the bit map up fromt.
41 // Later on we'll try to be more parsimonious with swap.
42 guarantee(_virtual_space.initialize(brs, brs.size()),
43 "couldn't reseve backing store for CMS bit map");
44 assert(_virtual_space.committed_size() == brs.size(),
45 "didn't reserve backing store for all of CMS bit map?");
46 _bm.set_map((uintptr_t*)_virtual_space.low());
47 assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
48 _bmWordSize, "inconsistency in bit map sizing");
49 _bm.set_size(_bmWordSize >> _shifter);
50 }
52 HeapWord* CMBitMapRO::getNextMarkedWordAddress(HeapWord* addr,
53 HeapWord* limit) const {
54 // First we must round addr *up* to a possible object boundary.
55 addr = (HeapWord*)align_size_up((intptr_t)addr,
56 HeapWordSize << _shifter);
57 size_t addrOffset = heapWordToOffset(addr);
58 if (limit == NULL) limit = _bmStartWord + _bmWordSize;
59 size_t limitOffset = heapWordToOffset(limit);
60 size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset);
61 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
62 assert(nextAddr >= addr, "get_next_one postcondition");
63 assert(nextAddr == limit || isMarked(nextAddr),
64 "get_next_one postcondition");
65 return nextAddr;
66 }
68 HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(HeapWord* addr,
69 HeapWord* limit) const {
70 size_t addrOffset = heapWordToOffset(addr);
71 if (limit == NULL) limit = _bmStartWord + _bmWordSize;
72 size_t limitOffset = heapWordToOffset(limit);
73 size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset);
74 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
75 assert(nextAddr >= addr, "get_next_one postcondition");
76 assert(nextAddr == limit || !isMarked(nextAddr),
77 "get_next_one postcondition");
78 return nextAddr;
79 }
81 int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const {
82 assert((diff & ((1 << _shifter) - 1)) == 0, "argument check");
83 return (int) (diff >> _shifter);
84 }
86 bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) {
87 HeapWord* left = MAX2(_bmStartWord, mr.start());
88 HeapWord* right = MIN2(_bmStartWord + _bmWordSize, mr.end());
89 if (right > left) {
90 // Right-open interval [leftOffset, rightOffset).
91 return _bm.iterate(cl, heapWordToOffset(left), heapWordToOffset(right));
92 } else {
93 return true;
94 }
95 }
97 void CMBitMapRO::mostly_disjoint_range_union(BitMap* from_bitmap,
98 size_t from_start_index,
99 HeapWord* to_start_word,
100 size_t word_num) {
101 _bm.mostly_disjoint_range_union(from_bitmap,
102 from_start_index,
103 heapWordToOffset(to_start_word),
104 word_num);
105 }
107 #ifndef PRODUCT
108 bool CMBitMapRO::covers(ReservedSpace rs) const {
109 // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
110 assert(((size_t)_bm.size() * (size_t)(1 << _shifter)) == _bmWordSize,
111 "size inconsistency");
112 return _bmStartWord == (HeapWord*)(rs.base()) &&
113 _bmWordSize == rs.size()>>LogHeapWordSize;
114 }
115 #endif
117 void CMBitMap::clearAll() {
118 _bm.clear();
119 return;
120 }
122 void CMBitMap::markRange(MemRegion mr) {
123 mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
124 assert(!mr.is_empty(), "unexpected empty region");
125 assert((offsetToHeapWord(heapWordToOffset(mr.end())) ==
126 ((HeapWord *) mr.end())),
127 "markRange memory region end is not card aligned");
128 // convert address range into offset range
129 _bm.at_put_range(heapWordToOffset(mr.start()),
130 heapWordToOffset(mr.end()), true);
131 }
133 void CMBitMap::clearRange(MemRegion mr) {
134 mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
135 assert(!mr.is_empty(), "unexpected empty region");
136 // convert address range into offset range
137 _bm.at_put_range(heapWordToOffset(mr.start()),
138 heapWordToOffset(mr.end()), false);
139 }
141 MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr,
142 HeapWord* end_addr) {
143 HeapWord* start = getNextMarkedWordAddress(addr);
144 start = MIN2(start, end_addr);
145 HeapWord* end = getNextUnmarkedWordAddress(start);
146 end = MIN2(end, end_addr);
147 assert(start <= end, "Consistency check");
148 MemRegion mr(start, end);
149 if (!mr.is_empty()) {
150 clearRange(mr);
151 }
152 return mr;
153 }
155 CMMarkStack::CMMarkStack(ConcurrentMark* cm) :
156 _base(NULL), _cm(cm)
157 #ifdef ASSERT
158 , _drain_in_progress(false)
159 , _drain_in_progress_yields(false)
160 #endif
161 {}
163 void CMMarkStack::allocate(size_t size) {
164 _base = NEW_C_HEAP_ARRAY(oop, size);
165 if (_base == NULL)
166 vm_exit_during_initialization("Failed to allocate "
167 "CM region mark stack");
168 _index = 0;
169 // QQQQ cast ...
170 _capacity = (jint) size;
171 _oops_do_bound = -1;
172 NOT_PRODUCT(_max_depth = 0);
173 }
175 CMMarkStack::~CMMarkStack() {
176 if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base);
177 }
179 void CMMarkStack::par_push(oop ptr) {
180 while (true) {
181 if (isFull()) {
182 _overflow = true;
183 return;
184 }
185 // Otherwise...
186 jint index = _index;
187 jint next_index = index+1;
188 jint res = Atomic::cmpxchg(next_index, &_index, index);
189 if (res == index) {
190 _base[index] = ptr;
191 // Note that we don't maintain this atomically. We could, but it
192 // doesn't seem necessary.
193 NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index));
194 return;
195 }
196 // Otherwise, we need to try again.
197 }
198 }
200 void CMMarkStack::par_adjoin_arr(oop* ptr_arr, int n) {
201 while (true) {
202 if (isFull()) {
203 _overflow = true;
204 return;
205 }
206 // Otherwise...
207 jint index = _index;
208 jint next_index = index + n;
209 if (next_index > _capacity) {
210 _overflow = true;
211 return;
212 }
213 jint res = Atomic::cmpxchg(next_index, &_index, index);
214 if (res == index) {
215 for (int i = 0; i < n; i++) {
216 int ind = index + i;
217 assert(ind < _capacity, "By overflow test above.");
218 _base[ind] = ptr_arr[i];
219 }
220 NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index));
221 return;
222 }
223 // Otherwise, we need to try again.
224 }
225 }
228 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
229 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
230 jint start = _index;
231 jint next_index = start + n;
232 if (next_index > _capacity) {
233 _overflow = true;
234 return;
235 }
236 // Otherwise.
237 _index = next_index;
238 for (int i = 0; i < n; i++) {
239 int ind = start + i;
240 guarantee(ind < _capacity, "By overflow test above.");
241 _base[ind] = ptr_arr[i];
242 }
243 }
246 bool CMMarkStack::par_pop_arr(oop* ptr_arr, int max, int* n) {
247 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
248 jint index = _index;
249 if (index == 0) {
250 *n = 0;
251 return false;
252 } else {
253 int k = MIN2(max, index);
254 jint new_ind = index - k;
255 for (int j = 0; j < k; j++) {
256 ptr_arr[j] = _base[new_ind + j];
257 }
258 _index = new_ind;
259 *n = k;
260 return true;
261 }
262 }
265 CMRegionStack::CMRegionStack() : _base(NULL) {}
267 void CMRegionStack::allocate(size_t size) {
268 _base = NEW_C_HEAP_ARRAY(MemRegion, size);
269 if (_base == NULL)
270 vm_exit_during_initialization("Failed to allocate "
271 "CM region mark stack");
272 _index = 0;
273 // QQQQ cast ...
274 _capacity = (jint) size;
275 }
277 CMRegionStack::~CMRegionStack() {
278 if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base);
279 }
281 void CMRegionStack::push(MemRegion mr) {
282 assert(mr.word_size() > 0, "Precondition");
283 while (true) {
284 if (isFull()) {
285 _overflow = true;
286 return;
287 }
288 // Otherwise...
289 jint index = _index;
290 jint next_index = index+1;
291 jint res = Atomic::cmpxchg(next_index, &_index, index);
292 if (res == index) {
293 _base[index] = mr;
294 return;
295 }
296 // Otherwise, we need to try again.
297 }
298 }
300 MemRegion CMRegionStack::pop() {
301 while (true) {
302 // Otherwise...
303 jint index = _index;
305 if (index == 0) {
306 return MemRegion();
307 }
308 jint next_index = index-1;
309 jint res = Atomic::cmpxchg(next_index, &_index, index);
310 if (res == index) {
311 MemRegion mr = _base[next_index];
312 if (mr.start() != NULL) {
313 tmp_guarantee_CM( mr.end() != NULL, "invariant" );
314 tmp_guarantee_CM( mr.word_size() > 0, "invariant" );
315 return mr;
316 } else {
317 // that entry was invalidated... let's skip it
318 tmp_guarantee_CM( mr.end() == NULL, "invariant" );
319 }
320 }
321 // Otherwise, we need to try again.
322 }
323 }
325 bool CMRegionStack::invalidate_entries_into_cset() {
326 bool result = false;
327 G1CollectedHeap* g1h = G1CollectedHeap::heap();
328 for (int i = 0; i < _oops_do_bound; ++i) {
329 MemRegion mr = _base[i];
330 if (mr.start() != NULL) {
331 tmp_guarantee_CM( mr.end() != NULL, "invariant");
332 tmp_guarantee_CM( mr.word_size() > 0, "invariant" );
333 HeapRegion* hr = g1h->heap_region_containing(mr.start());
334 tmp_guarantee_CM( hr != NULL, "invariant" );
335 if (hr->in_collection_set()) {
336 // The region points into the collection set
337 _base[i] = MemRegion();
338 result = true;
339 }
340 } else {
341 // that entry was invalidated... let's skip it
342 tmp_guarantee_CM( mr.end() == NULL, "invariant" );
343 }
344 }
345 return result;
346 }
348 template<class OopClosureClass>
349 bool CMMarkStack::drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after) {
350 assert(!_drain_in_progress || !_drain_in_progress_yields || yield_after
351 || SafepointSynchronize::is_at_safepoint(),
352 "Drain recursion must be yield-safe.");
353 bool res = true;
354 debug_only(_drain_in_progress = true);
355 debug_only(_drain_in_progress_yields = yield_after);
356 while (!isEmpty()) {
357 oop newOop = pop();
358 assert(G1CollectedHeap::heap()->is_in_reserved(newOop), "Bad pop");
359 assert(newOop->is_oop(), "Expected an oop");
360 assert(bm == NULL || bm->isMarked((HeapWord*)newOop),
361 "only grey objects on this stack");
362 // iterate over the oops in this oop, marking and pushing
363 // the ones in CMS generation.
364 newOop->oop_iterate(cl);
365 if (yield_after && _cm->do_yield_check()) {
366 res = false; break;
367 }
368 }
369 debug_only(_drain_in_progress = false);
370 return res;
371 }
373 void CMMarkStack::oops_do(OopClosure* f) {
374 if (_index == 0) return;
375 assert(_oops_do_bound != -1 && _oops_do_bound <= _index,
376 "Bound must be set.");
377 for (int i = 0; i < _oops_do_bound; i++) {
378 f->do_oop(&_base[i]);
379 }
380 _oops_do_bound = -1;
381 }
383 bool ConcurrentMark::not_yet_marked(oop obj) const {
384 return (_g1h->is_obj_ill(obj)
385 || (_g1h->is_in_permanent(obj)
386 && !nextMarkBitMap()->isMarked((HeapWord*)obj)));
387 }
389 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
390 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
391 #endif // _MSC_VER
393 ConcurrentMark::ConcurrentMark(ReservedSpace rs,
394 int max_regions) :
395 _markBitMap1(rs, MinObjAlignment - 1),
396 _markBitMap2(rs, MinObjAlignment - 1),
398 _parallel_marking_threads(0),
399 _sleep_factor(0.0),
400 _marking_task_overhead(1.0),
401 _cleanup_sleep_factor(0.0),
402 _cleanup_task_overhead(1.0),
403 _region_bm(max_regions, false /* in_resource_area*/),
404 _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >>
405 CardTableModRefBS::card_shift,
406 false /* in_resource_area*/),
407 _prevMarkBitMap(&_markBitMap1),
408 _nextMarkBitMap(&_markBitMap2),
409 _at_least_one_mark_complete(false),
411 _markStack(this),
412 _regionStack(),
413 // _finger set in set_non_marking_state
415 _max_task_num(MAX2(ParallelGCThreads, (size_t)1)),
416 // _active_tasks set in set_non_marking_state
417 // _tasks set inside the constructor
418 _task_queues(new CMTaskQueueSet((int) _max_task_num)),
419 _terminator(ParallelTaskTerminator((int) _max_task_num, _task_queues)),
421 _has_overflown(false),
422 _concurrent(false),
423 _has_aborted(false),
424 _restart_for_overflow(false),
425 _concurrent_marking_in_progress(false),
426 _should_gray_objects(false),
428 // _verbose_level set below
430 _init_times(),
431 _remark_times(), _remark_mark_times(), _remark_weak_ref_times(),
432 _cleanup_times(),
433 _total_counting_time(0.0),
434 _total_rs_scrub_time(0.0),
436 _parallel_workers(NULL),
437 _cleanup_co_tracker(G1CLGroup)
438 {
439 CMVerboseLevel verbose_level =
440 (CMVerboseLevel) G1MarkingVerboseLevel;
441 if (verbose_level < no_verbose)
442 verbose_level = no_verbose;
443 if (verbose_level > high_verbose)
444 verbose_level = high_verbose;
445 _verbose_level = verbose_level;
447 if (verbose_low())
448 gclog_or_tty->print_cr("[global] init, heap start = "PTR_FORMAT", "
449 "heap end = "PTR_FORMAT, _heap_start, _heap_end);
451 _markStack.allocate(G1MarkStackSize);
452 _regionStack.allocate(G1MarkRegionStackSize);
454 // Create & start a ConcurrentMark thread.
455 if (G1ConcMark) {
456 _cmThread = new ConcurrentMarkThread(this);
457 assert(cmThread() != NULL, "CM Thread should have been created");
458 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
459 } else {
460 _cmThread = NULL;
461 }
462 _g1h = G1CollectedHeap::heap();
463 assert(CGC_lock != NULL, "Where's the CGC_lock?");
464 assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency");
465 assert(_markBitMap2.covers(rs), "_markBitMap2 inconsistency");
467 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
468 satb_qs.set_buffer_size(G1SATBLogBufferSize);
470 int size = (int) MAX2(ParallelGCThreads, (size_t)1);
471 _par_cleanup_thread_state = NEW_C_HEAP_ARRAY(ParCleanupThreadState*, size);
472 for (int i = 0 ; i < size; i++) {
473 _par_cleanup_thread_state[i] = new ParCleanupThreadState;
474 }
476 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num);
477 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num);
479 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
480 _active_tasks = _max_task_num;
481 for (int i = 0; i < (int) _max_task_num; ++i) {
482 CMTaskQueue* task_queue = new CMTaskQueue();
483 task_queue->initialize();
484 _task_queues->register_queue(i, task_queue);
486 _tasks[i] = new CMTask(i, this, task_queue, _task_queues);
487 _accum_task_vtime[i] = 0.0;
488 }
490 if (ParallelMarkingThreads > ParallelGCThreads) {
491 vm_exit_during_initialization("Can't have more ParallelMarkingThreads "
492 "than ParallelGCThreads.");
493 }
494 if (ParallelGCThreads == 0) {
495 // if we are not running with any parallel GC threads we will not
496 // spawn any marking threads either
497 _parallel_marking_threads = 0;
498 _sleep_factor = 0.0;
499 _marking_task_overhead = 1.0;
500 } else {
501 if (ParallelMarkingThreads > 0) {
502 // notice that ParallelMarkingThreads overwrites G1MarkingOverheadPercent
503 // if both are set
505 _parallel_marking_threads = ParallelMarkingThreads;
506 _sleep_factor = 0.0;
507 _marking_task_overhead = 1.0;
508 } else if (G1MarkingOverheadPercent > 0) {
509 // we will calculate the number of parallel marking threads
510 // based on a target overhead with respect to the soft real-time
511 // goal
513 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0;
514 double overall_cm_overhead =
515 (double) MaxGCPauseMillis * marking_overhead /
516 (double) GCPauseIntervalMillis;
517 double cpu_ratio = 1.0 / (double) os::processor_count();
518 double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio);
519 double marking_task_overhead =
520 overall_cm_overhead / marking_thread_num *
521 (double) os::processor_count();
522 double sleep_factor =
523 (1.0 - marking_task_overhead) / marking_task_overhead;
525 _parallel_marking_threads = (size_t) marking_thread_num;
526 _sleep_factor = sleep_factor;
527 _marking_task_overhead = marking_task_overhead;
528 } else {
529 _parallel_marking_threads = MAX2((ParallelGCThreads + 2) / 4, (size_t)1);
530 _sleep_factor = 0.0;
531 _marking_task_overhead = 1.0;
532 }
534 if (parallel_marking_threads() > 1)
535 _cleanup_task_overhead = 1.0;
536 else
537 _cleanup_task_overhead = marking_task_overhead();
538 _cleanup_sleep_factor =
539 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
541 #if 0
542 gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads());
543 gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead());
544 gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor());
545 gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead());
546 gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor());
547 #endif
549 guarantee( parallel_marking_threads() > 0, "peace of mind" );
550 _parallel_workers = new WorkGang("Parallel Marking Threads",
551 (int) parallel_marking_threads(), false, true);
552 if (_parallel_workers == NULL)
553 vm_exit_during_initialization("Failed necessary allocation.");
554 }
556 // so that the call below can read a sensible value
557 _heap_start = (HeapWord*) rs.base();
558 set_non_marking_state();
559 }
561 void ConcurrentMark::update_g1_committed(bool force) {
562 // If concurrent marking is not in progress, then we do not need to
563 // update _heap_end. This has a subtle and important
564 // side-effect. Imagine that two evacuation pauses happen between
565 // marking completion and remark. The first one can grow the
566 // heap (hence now the finger is below the heap end). Then, the
567 // second one could unnecessarily push regions on the region
568 // stack. This causes the invariant that the region stack is empty
569 // at the beginning of remark to be false. By ensuring that we do
570 // not observe heap expansions after marking is complete, then we do
571 // not have this problem.
572 if (!concurrent_marking_in_progress() && !force)
573 return;
575 MemRegion committed = _g1h->g1_committed();
576 tmp_guarantee_CM( committed.start() == _heap_start,
577 "start shouldn't change" );
578 HeapWord* new_end = committed.end();
579 if (new_end > _heap_end) {
580 // The heap has been expanded.
582 _heap_end = new_end;
583 }
584 // Notice that the heap can also shrink. However, this only happens
585 // during a Full GC (at least currently) and the entire marking
586 // phase will bail out and the task will not be restarted. So, let's
587 // do nothing.
588 }
590 void ConcurrentMark::reset() {
591 // Starting values for these two. This should be called in a STW
592 // phase. CM will be notified of any future g1_committed expansions
593 // will be at the end of evacuation pauses, when tasks are
594 // inactive.
595 MemRegion committed = _g1h->g1_committed();
596 _heap_start = committed.start();
597 _heap_end = committed.end();
599 guarantee( _heap_start != NULL &&
600 _heap_end != NULL &&
601 _heap_start < _heap_end, "heap bounds should look ok" );
603 // reset all the marking data structures and any necessary flags
604 clear_marking_state();
606 if (verbose_low())
607 gclog_or_tty->print_cr("[global] resetting");
609 // We do reset all of them, since different phases will use
610 // different number of active threads. So, it's easiest to have all
611 // of them ready.
612 for (int i = 0; i < (int) _max_task_num; ++i)
613 _tasks[i]->reset(_nextMarkBitMap);
615 // we need this to make sure that the flag is on during the evac
616 // pause with initial mark piggy-backed
617 set_concurrent_marking_in_progress();
618 }
620 void ConcurrentMark::set_phase(size_t active_tasks, bool concurrent) {
621 guarantee( active_tasks <= _max_task_num, "we should not have more" );
623 _active_tasks = active_tasks;
624 // Need to update the three data structures below according to the
625 // number of active threads for this phase.
626 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues);
627 _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
628 _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
630 _concurrent = concurrent;
631 // We propagate this to all tasks, not just the active ones.
632 for (int i = 0; i < (int) _max_task_num; ++i)
633 _tasks[i]->set_concurrent(concurrent);
635 if (concurrent) {
636 set_concurrent_marking_in_progress();
637 } else {
638 // We currently assume that the concurrent flag has been set to
639 // false before we start remark. At this point we should also be
640 // in a STW phase.
641 guarantee( !concurrent_marking_in_progress(), "invariant" );
642 guarantee( _finger == _heap_end, "only way to get here" );
643 update_g1_committed(true);
644 }
645 }
647 void ConcurrentMark::set_non_marking_state() {
648 // We set the global marking state to some default values when we're
649 // not doing marking.
650 clear_marking_state();
651 _active_tasks = 0;
652 clear_concurrent_marking_in_progress();
653 }
655 ConcurrentMark::~ConcurrentMark() {
656 int size = (int) MAX2(ParallelGCThreads, (size_t)1);
657 for (int i = 0; i < size; i++) delete _par_cleanup_thread_state[i];
658 FREE_C_HEAP_ARRAY(ParCleanupThreadState*,
659 _par_cleanup_thread_state);
661 for (int i = 0; i < (int) _max_task_num; ++i) {
662 delete _task_queues->queue(i);
663 delete _tasks[i];
664 }
665 delete _task_queues;
666 FREE_C_HEAP_ARRAY(CMTask*, _max_task_num);
667 }
669 // This closure is used to mark refs into the g1 generation
670 // from external roots in the CMS bit map.
671 // Called at the first checkpoint.
672 //
674 #define PRINT_REACHABLE_AT_INITIAL_MARK 0
675 #if PRINT_REACHABLE_AT_INITIAL_MARK
676 static FILE* reachable_file = NULL;
678 class PrintReachableClosure: public OopsInGenClosure {
679 CMBitMap* _bm;
680 int _level;
681 public:
682 PrintReachableClosure(CMBitMap* bm) :
683 _bm(bm), _level(0) {
684 guarantee(reachable_file != NULL, "pre-condition");
685 }
686 void do_oop(oop* p) {
687 oop obj = *p;
688 HeapWord* obj_addr = (HeapWord*)obj;
689 if (obj == NULL) return;
690 fprintf(reachable_file, "%d: "PTR_FORMAT" -> "PTR_FORMAT" (%d)\n",
691 _level, p, (void*) obj, _bm->isMarked(obj_addr));
692 if (!_bm->isMarked(obj_addr)) {
693 _bm->mark(obj_addr);
694 _level++;
695 obj->oop_iterate(this);
696 _level--;
697 }
698 }
699 };
700 #endif // PRINT_REACHABLE_AT_INITIAL_MARK
702 #define SEND_HEAP_DUMP_TO_FILE 0
703 #if SEND_HEAP_DUMP_TO_FILE
704 static FILE* heap_dump_file = NULL;
705 #endif // SEND_HEAP_DUMP_TO_FILE
707 void ConcurrentMark::clearNextBitmap() {
708 guarantee(!G1CollectedHeap::heap()->mark_in_progress(), "Precondition.");
710 // clear the mark bitmap (no grey objects to start with).
711 // We need to do this in chunks and offer to yield in between
712 // each chunk.
713 HeapWord* start = _nextMarkBitMap->startWord();
714 HeapWord* end = _nextMarkBitMap->endWord();
715 HeapWord* cur = start;
716 size_t chunkSize = M;
717 while (cur < end) {
718 HeapWord* next = cur + chunkSize;
719 if (next > end)
720 next = end;
721 MemRegion mr(cur,next);
722 _nextMarkBitMap->clearRange(mr);
723 cur = next;
724 do_yield_check();
725 }
726 }
728 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
729 public:
730 bool doHeapRegion(HeapRegion* r) {
731 if (!r->continuesHumongous()) {
732 r->note_start_of_marking(true);
733 }
734 return false;
735 }
736 };
738 void ConcurrentMark::checkpointRootsInitialPre() {
739 G1CollectedHeap* g1h = G1CollectedHeap::heap();
740 G1CollectorPolicy* g1p = g1h->g1_policy();
742 _has_aborted = false;
744 // Find all the reachable objects...
745 #if PRINT_REACHABLE_AT_INITIAL_MARK
746 guarantee(reachable_file == NULL, "Protocol");
747 char fn_buf[100];
748 sprintf(fn_buf, "/tmp/reachable.txt.%d", os::current_process_id());
749 reachable_file = fopen(fn_buf, "w");
750 // clear the mark bitmap (no grey objects to start with)
751 _nextMarkBitMap->clearAll();
752 PrintReachableClosure prcl(_nextMarkBitMap);
753 g1h->process_strong_roots(
754 false, // fake perm gen collection
755 SharedHeap::SO_AllClasses,
756 &prcl, // Regular roots
757 &prcl // Perm Gen Roots
758 );
759 // The root iteration above "consumed" dirty cards in the perm gen.
760 // Therefore, as a shortcut, we dirty all such cards.
761 g1h->rem_set()->invalidate(g1h->perm_gen()->used_region(), false);
762 fclose(reachable_file);
763 reachable_file = NULL;
764 // clear the mark bitmap again.
765 _nextMarkBitMap->clearAll();
766 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
767 COMPILER2_PRESENT(DerivedPointerTable::clear());
768 #endif // PRINT_REACHABLE_AT_INITIAL_MARK
770 // Initialise marking structures. This has to be done in a STW phase.
771 reset();
772 }
774 class CMMarkRootsClosure: public OopsInGenClosure {
775 private:
776 ConcurrentMark* _cm;
777 G1CollectedHeap* _g1h;
778 bool _do_barrier;
780 public:
781 CMMarkRootsClosure(ConcurrentMark* cm,
782 G1CollectedHeap* g1h,
783 bool do_barrier) : _cm(cm), _g1h(g1h),
784 _do_barrier(do_barrier) { }
786 virtual void do_oop(narrowOop* p) {
787 guarantee(false, "NYI");
788 }
790 virtual void do_oop(oop* p) {
791 oop thisOop = *p;
792 if (thisOop != NULL) {
793 assert(thisOop->is_oop() || thisOop->mark() == NULL,
794 "expected an oop, possibly with mark word displaced");
795 HeapWord* addr = (HeapWord*)thisOop;
796 if (_g1h->is_in_g1_reserved(addr)) {
797 _cm->grayRoot(thisOop);
798 }
799 }
800 if (_do_barrier) {
801 assert(!_g1h->is_in_g1_reserved(p),
802 "Should be called on external roots");
803 do_barrier(p);
804 }
805 }
806 };
808 void ConcurrentMark::checkpointRootsInitialPost() {
809 G1CollectedHeap* g1h = G1CollectedHeap::heap();
811 // For each region note start of marking.
812 NoteStartOfMarkHRClosure startcl;
813 g1h->heap_region_iterate(&startcl);
815 // Start weak-reference discovery.
816 ReferenceProcessor* rp = g1h->ref_processor();
817 rp->verify_no_references_recorded();
818 rp->enable_discovery(); // enable ("weak") refs discovery
819 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
821 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
822 satb_mq_set.set_process_completed_threshold(G1SATBProcessCompletedThreshold);
823 satb_mq_set.set_active_all_threads(true);
825 // update_g1_committed() will be called at the end of an evac pause
826 // when marking is on. So, it's also called at the end of the
827 // initial-mark pause to update the heap end, if the heap expands
828 // during it. No need to call it here.
830 guarantee( !_cleanup_co_tracker.enabled(), "invariant" );
832 size_t max_marking_threads =
833 MAX2((size_t) 1, parallel_marking_threads());
834 for (int i = 0; i < (int)_max_task_num; ++i) {
835 _tasks[i]->enable_co_tracker();
836 if (i < (int) max_marking_threads)
837 _tasks[i]->reset_co_tracker(marking_task_overhead());
838 else
839 _tasks[i]->reset_co_tracker(0.0);
840 }
841 }
843 // Checkpoint the roots into this generation from outside
844 // this generation. [Note this initial checkpoint need only
845 // be approximate -- we'll do a catch up phase subsequently.]
846 void ConcurrentMark::checkpointRootsInitial() {
847 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
848 G1CollectedHeap* g1h = G1CollectedHeap::heap();
850 double start = os::elapsedTime();
851 GCOverheadReporter::recordSTWStart(start);
853 // If there has not been a GC[n-1] since last GC[n] cycle completed,
854 // precede our marking with a collection of all
855 // younger generations to keep floating garbage to a minimum.
856 // YSR: we won't do this for now -- it's an optimization to be
857 // done post-beta.
859 // YSR: ignoring weak refs for now; will do at bug fixing stage
860 // EVM: assert(discoveredRefsAreClear());
863 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
864 g1p->record_concurrent_mark_init_start();
865 checkpointRootsInitialPre();
867 // YSR: when concurrent precleaning is in place, we'll
868 // need to clear the cached card table here
870 ResourceMark rm;
871 HandleMark hm;
873 g1h->ensure_parsability(false);
874 g1h->perm_gen()->save_marks();
876 CMMarkRootsClosure notOlder(this, g1h, false);
877 CMMarkRootsClosure older(this, g1h, true);
879 g1h->set_marking_started();
880 g1h->rem_set()->prepare_for_younger_refs_iterate(false);
882 g1h->process_strong_roots(false, // fake perm gen collection
883 SharedHeap::SO_AllClasses,
884 ¬Older, // Regular roots
885 &older // Perm Gen Roots
886 );
887 checkpointRootsInitialPost();
889 // Statistics.
890 double end = os::elapsedTime();
891 _init_times.add((end - start) * 1000.0);
892 GCOverheadReporter::recordSTWEnd(end);
894 g1p->record_concurrent_mark_init_end();
895 }
897 /*
898 Notice that in the next two methods, we actually leave the STS
899 during the barrier sync and join it immediately afterwards. If we
900 do not do this, this then the following deadlock can occur: one
901 thread could be in the barrier sync code, waiting for the other
902 thread to also sync up, whereas another one could be trying to
903 yield, while also waiting for the other threads to sync up too.
905 Because the thread that does the sync barrier has left the STS, it
906 is possible to be suspended for a Full GC or an evacuation pause
907 could occur. This is actually safe, since the entering the sync
908 barrier is one of the last things do_marking_step() does, and it
909 doesn't manipulate any data structures afterwards.
910 */
912 void ConcurrentMark::enter_first_sync_barrier(int task_num) {
913 if (verbose_low())
914 gclog_or_tty->print_cr("[%d] entering first barrier", task_num);
916 ConcurrentGCThread::stsLeave();
917 _first_overflow_barrier_sync.enter();
918 ConcurrentGCThread::stsJoin();
919 // at this point everyone should have synced up and not be doing any
920 // more work
922 if (verbose_low())
923 gclog_or_tty->print_cr("[%d] leaving first barrier", task_num);
925 // let task 0 do this
926 if (task_num == 0) {
927 // task 0 is responsible for clearing the global data structures
928 clear_marking_state();
930 if (PrintGC) {
931 gclog_or_tty->date_stamp(PrintGCDateStamps);
932 gclog_or_tty->stamp(PrintGCTimeStamps);
933 gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]");
934 }
935 }
937 // after this, each task should reset its own data structures then
938 // then go into the second barrier
939 }
941 void ConcurrentMark::enter_second_sync_barrier(int task_num) {
942 if (verbose_low())
943 gclog_or_tty->print_cr("[%d] entering second barrier", task_num);
945 ConcurrentGCThread::stsLeave();
946 _second_overflow_barrier_sync.enter();
947 ConcurrentGCThread::stsJoin();
948 // at this point everything should be re-initialised and ready to go
950 if (verbose_low())
951 gclog_or_tty->print_cr("[%d] leaving second barrier", task_num);
952 }
954 void ConcurrentMark::grayRoot(oop p) {
955 HeapWord* addr = (HeapWord*) p;
956 // We can't really check against _heap_start and _heap_end, since it
957 // is possible during an evacuation pause with piggy-backed
958 // initial-mark that the committed space is expanded during the
959 // pause without CM observing this change. So the assertions below
960 // is a bit conservative; but better than nothing.
961 tmp_guarantee_CM( _g1h->g1_committed().contains(addr),
962 "address should be within the heap bounds" );
964 if (!_nextMarkBitMap->isMarked(addr))
965 _nextMarkBitMap->parMark(addr);
966 }
968 void ConcurrentMark::grayRegionIfNecessary(MemRegion mr) {
969 // The objects on the region have already been marked "in bulk" by
970 // the caller. We only need to decide whether to push the region on
971 // the region stack or not.
973 if (!concurrent_marking_in_progress() || !_should_gray_objects)
974 // We're done with marking and waiting for remark. We do not need to
975 // push anything else on the region stack.
976 return;
978 HeapWord* finger = _finger;
980 if (verbose_low())
981 gclog_or_tty->print_cr("[global] attempting to push "
982 "region ["PTR_FORMAT", "PTR_FORMAT"), finger is at "
983 PTR_FORMAT, mr.start(), mr.end(), finger);
985 if (mr.start() < finger) {
986 // The finger is always heap region aligned and it is not possible
987 // for mr to span heap regions.
988 tmp_guarantee_CM( mr.end() <= finger, "invariant" );
990 tmp_guarantee_CM( mr.start() <= mr.end() &&
991 _heap_start <= mr.start() &&
992 mr.end() <= _heap_end,
993 "region boundaries should fall within the committed space" );
994 if (verbose_low())
995 gclog_or_tty->print_cr("[global] region ["PTR_FORMAT", "PTR_FORMAT") "
996 "below the finger, pushing it",
997 mr.start(), mr.end());
999 if (!region_stack_push(mr)) {
1000 if (verbose_low())
1001 gclog_or_tty->print_cr("[global] region stack has overflown.");
1002 }
1003 }
1004 }
1006 void ConcurrentMark::markAndGrayObjectIfNecessary(oop p) {
1007 // The object is not marked by the caller. We need to at least mark
1008 // it and maybe push in on the stack.
1010 HeapWord* addr = (HeapWord*)p;
1011 if (!_nextMarkBitMap->isMarked(addr)) {
1012 // We definitely need to mark it, irrespective whether we bail out
1013 // because we're done with marking.
1014 if (_nextMarkBitMap->parMark(addr)) {
1015 if (!concurrent_marking_in_progress() || !_should_gray_objects)
1016 // If we're done with concurrent marking and we're waiting for
1017 // remark, then we're not pushing anything on the stack.
1018 return;
1020 // No OrderAccess:store_load() is needed. It is implicit in the
1021 // CAS done in parMark(addr) above
1022 HeapWord* finger = _finger;
1024 if (addr < finger) {
1025 if (!mark_stack_push(oop(addr))) {
1026 if (verbose_low())
1027 gclog_or_tty->print_cr("[global] global stack overflow "
1028 "during parMark");
1029 }
1030 }
1031 }
1032 }
1033 }
1035 class CMConcurrentMarkingTask: public AbstractGangTask {
1036 private:
1037 ConcurrentMark* _cm;
1038 ConcurrentMarkThread* _cmt;
1040 public:
1041 void work(int worker_i) {
1042 guarantee( Thread::current()->is_ConcurrentGC_thread(),
1043 "this should only be done by a conc GC thread" );
1045 double start_vtime = os::elapsedVTime();
1047 ConcurrentGCThread::stsJoin();
1049 guarantee( (size_t)worker_i < _cm->active_tasks(), "invariant" );
1050 CMTask* the_task = _cm->task(worker_i);
1051 the_task->start_co_tracker();
1052 the_task->record_start_time();
1053 if (!_cm->has_aborted()) {
1054 do {
1055 double start_vtime_sec = os::elapsedVTime();
1056 double start_time_sec = os::elapsedTime();
1057 the_task->do_marking_step(10.0);
1058 double end_time_sec = os::elapsedTime();
1059 double end_vtime_sec = os::elapsedVTime();
1060 double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec;
1061 double elapsed_time_sec = end_time_sec - start_time_sec;
1062 _cm->clear_has_overflown();
1064 bool ret = _cm->do_yield_check(worker_i);
1066 jlong sleep_time_ms;
1067 if (!_cm->has_aborted() && the_task->has_aborted()) {
1068 sleep_time_ms =
1069 (jlong) (elapsed_vtime_sec * _cm->sleep_factor() * 1000.0);
1070 ConcurrentGCThread::stsLeave();
1071 os::sleep(Thread::current(), sleep_time_ms, false);
1072 ConcurrentGCThread::stsJoin();
1073 }
1074 double end_time2_sec = os::elapsedTime();
1075 double elapsed_time2_sec = end_time2_sec - start_time_sec;
1077 the_task->update_co_tracker();
1079 #if 0
1080 gclog_or_tty->print_cr("CM: elapsed %1.4lf ms, sleep %1.4lf ms, "
1081 "overhead %1.4lf",
1082 elapsed_vtime_sec * 1000.0, (double) sleep_time_ms,
1083 the_task->conc_overhead(os::elapsedTime()) * 8.0);
1084 gclog_or_tty->print_cr("elapsed time %1.4lf ms, time 2: %1.4lf ms",
1085 elapsed_time_sec * 1000.0, elapsed_time2_sec * 1000.0);
1086 #endif
1087 } while (!_cm->has_aborted() && the_task->has_aborted());
1088 }
1089 the_task->record_end_time();
1090 guarantee( !the_task->has_aborted() || _cm->has_aborted(), "invariant" );
1092 ConcurrentGCThread::stsLeave();
1094 double end_vtime = os::elapsedVTime();
1095 the_task->update_co_tracker(true);
1096 _cm->update_accum_task_vtime(worker_i, end_vtime - start_vtime);
1097 }
1099 CMConcurrentMarkingTask(ConcurrentMark* cm,
1100 ConcurrentMarkThread* cmt) :
1101 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { }
1103 ~CMConcurrentMarkingTask() { }
1104 };
1106 void ConcurrentMark::markFromRoots() {
1107 // we might be tempted to assert that:
1108 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1109 // "inconsistent argument?");
1110 // However that wouldn't be right, because it's possible that
1111 // a safepoint is indeed in progress as a younger generation
1112 // stop-the-world GC happens even as we mark in this generation.
1114 _restart_for_overflow = false;
1116 set_phase(MAX2((size_t) 1, parallel_marking_threads()), true);
1118 CMConcurrentMarkingTask markingTask(this, cmThread());
1119 if (parallel_marking_threads() > 0)
1120 _parallel_workers->run_task(&markingTask);
1121 else
1122 markingTask.work(0);
1123 print_stats();
1124 }
1126 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1127 // world is stopped at this checkpoint
1128 assert(SafepointSynchronize::is_at_safepoint(),
1129 "world should be stopped");
1130 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1132 // If a full collection has happened, we shouldn't do this.
1133 if (has_aborted()) {
1134 g1h->set_marking_complete(); // So bitmap clearing isn't confused
1135 return;
1136 }
1138 G1CollectorPolicy* g1p = g1h->g1_policy();
1139 g1p->record_concurrent_mark_remark_start();
1141 double start = os::elapsedTime();
1142 GCOverheadReporter::recordSTWStart(start);
1144 checkpointRootsFinalWork();
1146 double mark_work_end = os::elapsedTime();
1148 weakRefsWork(clear_all_soft_refs);
1150 if (has_overflown()) {
1151 // Oops. We overflowed. Restart concurrent marking.
1152 _restart_for_overflow = true;
1153 // Clear the flag. We do not need it any more.
1154 clear_has_overflown();
1155 if (G1TraceMarkStackOverflow)
1156 gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
1157 } else {
1158 // We're done with marking.
1159 JavaThread::satb_mark_queue_set().set_active_all_threads(false);
1161 if (VerifyDuringGC) {
1162 g1h->prepare_for_verify();
1163 g1h->verify(/* allow_dirty */ true,
1164 /* silent */ false,
1165 /* use_prev_marking */ false);
1166 }
1167 }
1169 #if VERIFY_OBJS_PROCESSED
1170 _scan_obj_cl.objs_processed = 0;
1171 ThreadLocalObjQueue::objs_enqueued = 0;
1172 #endif
1174 // Statistics
1175 double now = os::elapsedTime();
1176 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1177 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1178 _remark_times.add((now - start) * 1000.0);
1180 GCOverheadReporter::recordSTWEnd(now);
1181 for (int i = 0; i < (int)_max_task_num; ++i)
1182 _tasks[i]->disable_co_tracker();
1183 _cleanup_co_tracker.enable();
1184 _cleanup_co_tracker.reset(cleanup_task_overhead());
1185 g1p->record_concurrent_mark_remark_end();
1186 }
1189 #define CARD_BM_TEST_MODE 0
1191 class CalcLiveObjectsClosure: public HeapRegionClosure {
1193 CMBitMapRO* _bm;
1194 ConcurrentMark* _cm;
1195 COTracker* _co_tracker;
1196 bool _changed;
1197 bool _yield;
1198 size_t _words_done;
1199 size_t _tot_live;
1200 size_t _tot_used;
1201 size_t _regions_done;
1202 double _start_vtime_sec;
1204 BitMap* _region_bm;
1205 BitMap* _card_bm;
1206 intptr_t _bottom_card_num;
1207 bool _final;
1209 void mark_card_num_range(intptr_t start_card_num, intptr_t last_card_num) {
1210 for (intptr_t i = start_card_num; i <= last_card_num; i++) {
1211 #if CARD_BM_TEST_MODE
1212 guarantee(_card_bm->at(i - _bottom_card_num),
1213 "Should already be set.");
1214 #else
1215 _card_bm->par_at_put(i - _bottom_card_num, 1);
1216 #endif
1217 }
1218 }
1220 public:
1221 CalcLiveObjectsClosure(bool final,
1222 CMBitMapRO *bm, ConcurrentMark *cm,
1223 BitMap* region_bm, BitMap* card_bm,
1224 COTracker* co_tracker) :
1225 _bm(bm), _cm(cm), _changed(false), _yield(true),
1226 _words_done(0), _tot_live(0), _tot_used(0),
1227 _region_bm(region_bm), _card_bm(card_bm),
1228 _final(final), _co_tracker(co_tracker),
1229 _regions_done(0), _start_vtime_sec(0.0)
1230 {
1231 _bottom_card_num =
1232 intptr_t(uintptr_t(G1CollectedHeap::heap()->reserved_region().start()) >>
1233 CardTableModRefBS::card_shift);
1234 }
1236 // It takes a region that's not empty (i.e., it has at least one
1237 // live object in it and sets its corresponding bit on the region
1238 // bitmap to 1. If the region is "starts humongous" it will also set
1239 // to 1 the bits on the region bitmap that correspond to its
1240 // associated "continues humongous" regions.
1241 void set_bit_for_region(HeapRegion* hr) {
1242 assert(!hr->continuesHumongous(), "should have filtered those out");
1244 size_t index = hr->hrs_index();
1245 if (!hr->startsHumongous()) {
1246 // Normal (non-humongous) case: just set the bit.
1247 _region_bm->par_at_put((BitMap::idx_t) index, true);
1248 } else {
1249 // Starts humongous case: calculate how many regions are part of
1250 // this humongous region and then set the bit range. It might
1251 // have been a bit more efficient to look at the object that
1252 // spans these humongous regions to calculate their number from
1253 // the object's size. However, it's a good idea to calculate
1254 // this based on the metadata itself, and not the region
1255 // contents, so that this code is not aware of what goes into
1256 // the humongous regions (in case this changes in the future).
1257 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1258 size_t end_index = index + 1;
1259 while (end_index < g1h->n_regions()) {
1260 HeapRegion* chr = g1h->region_at(end_index);
1261 if (!chr->continuesHumongous()) {
1262 break;
1263 }
1264 end_index += 1;
1265 }
1266 _region_bm->par_at_put_range((BitMap::idx_t) index,
1267 (BitMap::idx_t) end_index, true);
1268 }
1269 }
1271 bool doHeapRegion(HeapRegion* hr) {
1272 if (_co_tracker != NULL)
1273 _co_tracker->update();
1275 if (!_final && _regions_done == 0)
1276 _start_vtime_sec = os::elapsedVTime();
1278 if (hr->continuesHumongous()) {
1279 // We will ignore these here and process them when their
1280 // associated "starts humongous" region is processed (see
1281 // set_bit_for_heap_region()). Note that we cannot rely on their
1282 // associated "starts humongous" region to have their bit set to
1283 // 1 since, due to the region chunking in the parallel region
1284 // iteration, a "continues humongous" region might be visited
1285 // before its associated "starts humongous".
1286 return false;
1287 }
1289 HeapWord* nextTop = hr->next_top_at_mark_start();
1290 HeapWord* start = hr->top_at_conc_mark_count();
1291 assert(hr->bottom() <= start && start <= hr->end() &&
1292 hr->bottom() <= nextTop && nextTop <= hr->end() &&
1293 start <= nextTop,
1294 "Preconditions.");
1295 // Otherwise, record the number of word's we'll examine.
1296 size_t words_done = (nextTop - start);
1297 // Find the first marked object at or after "start".
1298 start = _bm->getNextMarkedWordAddress(start, nextTop);
1299 size_t marked_bytes = 0;
1301 // Below, the term "card num" means the result of shifting an address
1302 // by the card shift -- address 0 corresponds to card number 0. One
1303 // must subtract the card num of the bottom of the heap to obtain a
1304 // card table index.
1305 // The first card num of the sequence of live cards currently being
1306 // constructed. -1 ==> no sequence.
1307 intptr_t start_card_num = -1;
1308 // The last card num of the sequence of live cards currently being
1309 // constructed. -1 ==> no sequence.
1310 intptr_t last_card_num = -1;
1312 while (start < nextTop) {
1313 if (_yield && _cm->do_yield_check()) {
1314 // We yielded. It might be for a full collection, in which case
1315 // all bets are off; terminate the traversal.
1316 if (_cm->has_aborted()) {
1317 _changed = false;
1318 return true;
1319 } else {
1320 // Otherwise, it might be a collection pause, and the region
1321 // we're looking at might be in the collection set. We'll
1322 // abandon this region.
1323 return false;
1324 }
1325 }
1326 oop obj = oop(start);
1327 int obj_sz = obj->size();
1328 // The card num of the start of the current object.
1329 intptr_t obj_card_num =
1330 intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
1332 HeapWord* obj_last = start + obj_sz - 1;
1333 intptr_t obj_last_card_num =
1334 intptr_t(uintptr_t(obj_last) >> CardTableModRefBS::card_shift);
1336 if (obj_card_num != last_card_num) {
1337 if (start_card_num == -1) {
1338 assert(last_card_num == -1, "Both or neither.");
1339 start_card_num = obj_card_num;
1340 } else {
1341 assert(last_card_num != -1, "Both or neither.");
1342 assert(obj_card_num >= last_card_num, "Inv");
1343 if ((obj_card_num - last_card_num) > 1) {
1344 // Mark the last run, and start a new one.
1345 mark_card_num_range(start_card_num, last_card_num);
1346 start_card_num = obj_card_num;
1347 }
1348 }
1349 #if CARD_BM_TEST_MODE
1350 /*
1351 gclog_or_tty->print_cr("Setting bits from %d/%d.",
1352 obj_card_num - _bottom_card_num,
1353 obj_last_card_num - _bottom_card_num);
1354 */
1355 for (intptr_t j = obj_card_num; j <= obj_last_card_num; j++) {
1356 _card_bm->par_at_put(j - _bottom_card_num, 1);
1357 }
1358 #endif
1359 }
1360 // In any case, we set the last card num.
1361 last_card_num = obj_last_card_num;
1363 marked_bytes += obj_sz * HeapWordSize;
1364 // Find the next marked object after this one.
1365 start = _bm->getNextMarkedWordAddress(start + 1, nextTop);
1366 _changed = true;
1367 }
1368 // Handle the last range, if any.
1369 if (start_card_num != -1)
1370 mark_card_num_range(start_card_num, last_card_num);
1371 if (_final) {
1372 // Mark the allocated-since-marking portion...
1373 HeapWord* tp = hr->top();
1374 if (nextTop < tp) {
1375 start_card_num =
1376 intptr_t(uintptr_t(nextTop) >> CardTableModRefBS::card_shift);
1377 last_card_num =
1378 intptr_t(uintptr_t(tp) >> CardTableModRefBS::card_shift);
1379 mark_card_num_range(start_card_num, last_card_num);
1380 // This definitely means the region has live objects.
1381 set_bit_for_region(hr);
1382 }
1383 }
1385 hr->add_to_marked_bytes(marked_bytes);
1386 // Update the live region bitmap.
1387 if (marked_bytes > 0) {
1388 set_bit_for_region(hr);
1389 }
1390 hr->set_top_at_conc_mark_count(nextTop);
1391 _tot_live += hr->next_live_bytes();
1392 _tot_used += hr->used();
1393 _words_done = words_done;
1395 if (!_final) {
1396 ++_regions_done;
1397 if (_regions_done % 10 == 0) {
1398 double end_vtime_sec = os::elapsedVTime();
1399 double elapsed_vtime_sec = end_vtime_sec - _start_vtime_sec;
1400 if (elapsed_vtime_sec > (10.0 / 1000.0)) {
1401 jlong sleep_time_ms =
1402 (jlong) (elapsed_vtime_sec * _cm->cleanup_sleep_factor() * 1000.0);
1403 #if 0
1404 gclog_or_tty->print_cr("CL: elapsed %1.4lf ms, sleep %1.4lf ms, "
1405 "overhead %1.4lf",
1406 elapsed_vtime_sec * 1000.0, (double) sleep_time_ms,
1407 _co_tracker->concOverhead(os::elapsedTime()));
1408 #endif
1409 os::sleep(Thread::current(), sleep_time_ms, false);
1410 _start_vtime_sec = end_vtime_sec;
1411 }
1412 }
1413 }
1415 return false;
1416 }
1418 bool changed() { return _changed; }
1419 void reset() { _changed = false; _words_done = 0; }
1420 void no_yield() { _yield = false; }
1421 size_t words_done() { return _words_done; }
1422 size_t tot_live() { return _tot_live; }
1423 size_t tot_used() { return _tot_used; }
1424 };
1427 void ConcurrentMark::calcDesiredRegions() {
1428 guarantee( _cleanup_co_tracker.enabled(), "invariant" );
1429 _cleanup_co_tracker.start();
1431 _region_bm.clear();
1432 _card_bm.clear();
1433 CalcLiveObjectsClosure calccl(false /*final*/,
1434 nextMarkBitMap(), this,
1435 &_region_bm, &_card_bm,
1436 &_cleanup_co_tracker);
1437 G1CollectedHeap *g1h = G1CollectedHeap::heap();
1438 g1h->heap_region_iterate(&calccl);
1440 do {
1441 calccl.reset();
1442 g1h->heap_region_iterate(&calccl);
1443 } while (calccl.changed());
1445 _cleanup_co_tracker.update(true);
1446 }
1448 class G1ParFinalCountTask: public AbstractGangTask {
1449 protected:
1450 G1CollectedHeap* _g1h;
1451 CMBitMap* _bm;
1452 size_t _n_workers;
1453 size_t *_live_bytes;
1454 size_t *_used_bytes;
1455 BitMap* _region_bm;
1456 BitMap* _card_bm;
1457 public:
1458 G1ParFinalCountTask(G1CollectedHeap* g1h, CMBitMap* bm,
1459 BitMap* region_bm, BitMap* card_bm) :
1460 AbstractGangTask("G1 final counting"), _g1h(g1h),
1461 _bm(bm), _region_bm(region_bm), _card_bm(card_bm)
1462 {
1463 if (ParallelGCThreads > 0)
1464 _n_workers = _g1h->workers()->total_workers();
1465 else
1466 _n_workers = 1;
1467 _live_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers);
1468 _used_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers);
1469 }
1471 ~G1ParFinalCountTask() {
1472 FREE_C_HEAP_ARRAY(size_t, _live_bytes);
1473 FREE_C_HEAP_ARRAY(size_t, _used_bytes);
1474 }
1476 void work(int i) {
1477 CalcLiveObjectsClosure calccl(true /*final*/,
1478 _bm, _g1h->concurrent_mark(),
1479 _region_bm, _card_bm,
1480 NULL /* CO tracker */);
1481 calccl.no_yield();
1482 if (ParallelGCThreads > 0) {
1483 _g1h->heap_region_par_iterate_chunked(&calccl, i,
1484 HeapRegion::FinalCountClaimValue);
1485 } else {
1486 _g1h->heap_region_iterate(&calccl);
1487 }
1488 assert(calccl.complete(), "Shouldn't have yielded!");
1490 guarantee( (size_t)i < _n_workers, "invariant" );
1491 _live_bytes[i] = calccl.tot_live();
1492 _used_bytes[i] = calccl.tot_used();
1493 }
1494 size_t live_bytes() {
1495 size_t live_bytes = 0;
1496 for (size_t i = 0; i < _n_workers; ++i)
1497 live_bytes += _live_bytes[i];
1498 return live_bytes;
1499 }
1500 size_t used_bytes() {
1501 size_t used_bytes = 0;
1502 for (size_t i = 0; i < _n_workers; ++i)
1503 used_bytes += _used_bytes[i];
1504 return used_bytes;
1505 }
1506 };
1508 class G1ParNoteEndTask;
1510 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1511 G1CollectedHeap* _g1;
1512 int _worker_num;
1513 size_t _max_live_bytes;
1514 size_t _regions_claimed;
1515 size_t _freed_bytes;
1516 size_t _cleared_h_regions;
1517 size_t _freed_regions;
1518 UncleanRegionList* _unclean_region_list;
1519 double _claimed_region_time;
1520 double _max_region_time;
1522 public:
1523 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1524 UncleanRegionList* list,
1525 int worker_num);
1526 size_t freed_bytes() { return _freed_bytes; }
1527 size_t cleared_h_regions() { return _cleared_h_regions; }
1528 size_t freed_regions() { return _freed_regions; }
1529 UncleanRegionList* unclean_region_list() {
1530 return _unclean_region_list;
1531 }
1533 bool doHeapRegion(HeapRegion *r);
1535 size_t max_live_bytes() { return _max_live_bytes; }
1536 size_t regions_claimed() { return _regions_claimed; }
1537 double claimed_region_time_sec() { return _claimed_region_time; }
1538 double max_region_time_sec() { return _max_region_time; }
1539 };
1541 class G1ParNoteEndTask: public AbstractGangTask {
1542 friend class G1NoteEndOfConcMarkClosure;
1543 protected:
1544 G1CollectedHeap* _g1h;
1545 size_t _max_live_bytes;
1546 size_t _freed_bytes;
1547 ConcurrentMark::ParCleanupThreadState** _par_cleanup_thread_state;
1548 public:
1549 G1ParNoteEndTask(G1CollectedHeap* g1h,
1550 ConcurrentMark::ParCleanupThreadState**
1551 par_cleanup_thread_state) :
1552 AbstractGangTask("G1 note end"), _g1h(g1h),
1553 _max_live_bytes(0), _freed_bytes(0),
1554 _par_cleanup_thread_state(par_cleanup_thread_state)
1555 {}
1557 void work(int i) {
1558 double start = os::elapsedTime();
1559 G1NoteEndOfConcMarkClosure g1_note_end(_g1h,
1560 &_par_cleanup_thread_state[i]->list,
1561 i);
1562 if (ParallelGCThreads > 0) {
1563 _g1h->heap_region_par_iterate_chunked(&g1_note_end, i,
1564 HeapRegion::NoteEndClaimValue);
1565 } else {
1566 _g1h->heap_region_iterate(&g1_note_end);
1567 }
1568 assert(g1_note_end.complete(), "Shouldn't have yielded!");
1570 // Now finish up freeing the current thread's regions.
1571 _g1h->finish_free_region_work(g1_note_end.freed_bytes(),
1572 g1_note_end.cleared_h_regions(),
1573 0, NULL);
1574 {
1575 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1576 _max_live_bytes += g1_note_end.max_live_bytes();
1577 _freed_bytes += g1_note_end.freed_bytes();
1578 }
1579 double end = os::elapsedTime();
1580 if (G1PrintParCleanupStats) {
1581 gclog_or_tty->print(" Worker thread %d [%8.3f..%8.3f = %8.3f ms] "
1582 "claimed %d regions (tot = %8.3f ms, max = %8.3f ms).\n",
1583 i, start, end, (end-start)*1000.0,
1584 g1_note_end.regions_claimed(),
1585 g1_note_end.claimed_region_time_sec()*1000.0,
1586 g1_note_end.max_region_time_sec()*1000.0);
1587 }
1588 }
1589 size_t max_live_bytes() { return _max_live_bytes; }
1590 size_t freed_bytes() { return _freed_bytes; }
1591 };
1593 class G1ParScrubRemSetTask: public AbstractGangTask {
1594 protected:
1595 G1RemSet* _g1rs;
1596 BitMap* _region_bm;
1597 BitMap* _card_bm;
1598 public:
1599 G1ParScrubRemSetTask(G1CollectedHeap* g1h,
1600 BitMap* region_bm, BitMap* card_bm) :
1601 AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()),
1602 _region_bm(region_bm), _card_bm(card_bm)
1603 {}
1605 void work(int i) {
1606 if (ParallelGCThreads > 0) {
1607 _g1rs->scrub_par(_region_bm, _card_bm, i,
1608 HeapRegion::ScrubRemSetClaimValue);
1609 } else {
1610 _g1rs->scrub(_region_bm, _card_bm);
1611 }
1612 }
1614 };
1616 G1NoteEndOfConcMarkClosure::
1617 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1618 UncleanRegionList* list,
1619 int worker_num)
1620 : _g1(g1), _worker_num(worker_num),
1621 _max_live_bytes(0), _regions_claimed(0),
1622 _freed_bytes(0), _cleared_h_regions(0), _freed_regions(0),
1623 _claimed_region_time(0.0), _max_region_time(0.0),
1624 _unclean_region_list(list)
1625 {}
1627 bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *r) {
1628 // We use a claim value of zero here because all regions
1629 // were claimed with value 1 in the FinalCount task.
1630 r->reset_gc_time_stamp();
1631 if (!r->continuesHumongous()) {
1632 double start = os::elapsedTime();
1633 _regions_claimed++;
1634 r->note_end_of_marking();
1635 _max_live_bytes += r->max_live_bytes();
1636 _g1->free_region_if_totally_empty_work(r,
1637 _freed_bytes,
1638 _cleared_h_regions,
1639 _freed_regions,
1640 _unclean_region_list,
1641 true /*par*/);
1642 double region_time = (os::elapsedTime() - start);
1643 _claimed_region_time += region_time;
1644 if (region_time > _max_region_time) _max_region_time = region_time;
1645 }
1646 return false;
1647 }
1649 void ConcurrentMark::cleanup() {
1650 // world is stopped at this checkpoint
1651 assert(SafepointSynchronize::is_at_safepoint(),
1652 "world should be stopped");
1653 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1655 // If a full collection has happened, we shouldn't do this.
1656 if (has_aborted()) {
1657 g1h->set_marking_complete(); // So bitmap clearing isn't confused
1658 return;
1659 }
1661 _cleanup_co_tracker.disable();
1663 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
1664 g1p->record_concurrent_mark_cleanup_start();
1666 double start = os::elapsedTime();
1667 GCOverheadReporter::recordSTWStart(start);
1669 // Do counting once more with the world stopped for good measure.
1670 G1ParFinalCountTask g1_par_count_task(g1h, nextMarkBitMap(),
1671 &_region_bm, &_card_bm);
1672 if (ParallelGCThreads > 0) {
1673 assert(g1h->check_heap_region_claim_values(
1674 HeapRegion::InitialClaimValue),
1675 "sanity check");
1677 int n_workers = g1h->workers()->total_workers();
1678 g1h->set_par_threads(n_workers);
1679 g1h->workers()->run_task(&g1_par_count_task);
1680 g1h->set_par_threads(0);
1682 assert(g1h->check_heap_region_claim_values(
1683 HeapRegion::FinalCountClaimValue),
1684 "sanity check");
1685 } else {
1686 g1_par_count_task.work(0);
1687 }
1689 size_t known_garbage_bytes =
1690 g1_par_count_task.used_bytes() - g1_par_count_task.live_bytes();
1691 #if 0
1692 gclog_or_tty->print_cr("used %1.2lf, live %1.2lf, garbage %1.2lf",
1693 (double) g1_par_count_task.used_bytes() / (double) (1024 * 1024),
1694 (double) g1_par_count_task.live_bytes() / (double) (1024 * 1024),
1695 (double) known_garbage_bytes / (double) (1024 * 1024));
1696 #endif // 0
1697 g1p->set_known_garbage_bytes(known_garbage_bytes);
1699 size_t start_used_bytes = g1h->used();
1700 _at_least_one_mark_complete = true;
1701 g1h->set_marking_complete();
1703 double count_end = os::elapsedTime();
1704 double this_final_counting_time = (count_end - start);
1705 if (G1PrintParCleanupStats) {
1706 gclog_or_tty->print_cr("Cleanup:");
1707 gclog_or_tty->print_cr(" Finalize counting: %8.3f ms",
1708 this_final_counting_time*1000.0);
1709 }
1710 _total_counting_time += this_final_counting_time;
1712 // Install newly created mark bitMap as "prev".
1713 swapMarkBitMaps();
1715 g1h->reset_gc_time_stamp();
1717 // Note end of marking in all heap regions.
1718 double note_end_start = os::elapsedTime();
1719 G1ParNoteEndTask g1_par_note_end_task(g1h, _par_cleanup_thread_state);
1720 if (ParallelGCThreads > 0) {
1721 int n_workers = g1h->workers()->total_workers();
1722 g1h->set_par_threads(n_workers);
1723 g1h->workers()->run_task(&g1_par_note_end_task);
1724 g1h->set_par_threads(0);
1726 assert(g1h->check_heap_region_claim_values(HeapRegion::NoteEndClaimValue),
1727 "sanity check");
1728 } else {
1729 g1_par_note_end_task.work(0);
1730 }
1731 g1h->set_unclean_regions_coming(true);
1732 double note_end_end = os::elapsedTime();
1733 // Tell the mutators that there might be unclean regions coming...
1734 if (G1PrintParCleanupStats) {
1735 gclog_or_tty->print_cr(" note end of marking: %8.3f ms.",
1736 (note_end_end - note_end_start)*1000.0);
1737 }
1740 // call below, since it affects the metric by which we sort the heap
1741 // regions.
1742 if (G1ScrubRemSets) {
1743 double rs_scrub_start = os::elapsedTime();
1744 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm);
1745 if (ParallelGCThreads > 0) {
1746 int n_workers = g1h->workers()->total_workers();
1747 g1h->set_par_threads(n_workers);
1748 g1h->workers()->run_task(&g1_par_scrub_rs_task);
1749 g1h->set_par_threads(0);
1751 assert(g1h->check_heap_region_claim_values(
1752 HeapRegion::ScrubRemSetClaimValue),
1753 "sanity check");
1754 } else {
1755 g1_par_scrub_rs_task.work(0);
1756 }
1758 double rs_scrub_end = os::elapsedTime();
1759 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
1760 _total_rs_scrub_time += this_rs_scrub_time;
1761 }
1763 // this will also free any regions totally full of garbage objects,
1764 // and sort the regions.
1765 g1h->g1_policy()->record_concurrent_mark_cleanup_end(
1766 g1_par_note_end_task.freed_bytes(),
1767 g1_par_note_end_task.max_live_bytes());
1769 // Statistics.
1770 double end = os::elapsedTime();
1771 _cleanup_times.add((end - start) * 1000.0);
1772 GCOverheadReporter::recordSTWEnd(end);
1774 // G1CollectedHeap::heap()->print();
1775 // gclog_or_tty->print_cr("HEAP GC TIME STAMP : %d",
1776 // G1CollectedHeap::heap()->get_gc_time_stamp());
1778 if (PrintGC || PrintGCDetails) {
1779 g1h->print_size_transition(gclog_or_tty,
1780 start_used_bytes,
1781 g1h->used(),
1782 g1h->capacity());
1783 }
1785 size_t cleaned_up_bytes = start_used_bytes - g1h->used();
1786 g1p->decrease_known_garbage_bytes(cleaned_up_bytes);
1788 // We need to make this be a "collection" so any collection pause that
1789 // races with it goes around and waits for completeCleanup to finish.
1790 g1h->increment_total_collections();
1792 if (VerifyDuringGC) {
1793 g1h->prepare_for_verify();
1794 g1h->verify(/* allow_dirty */ true,
1795 /* silent */ false,
1796 /* use_prev_marking */ true);
1797 }
1798 }
1800 void ConcurrentMark::completeCleanup() {
1801 // A full collection intervened.
1802 if (has_aborted()) return;
1804 int first = 0;
1805 int last = (int)MAX2(ParallelGCThreads, (size_t)1);
1806 for (int t = 0; t < last; t++) {
1807 UncleanRegionList* list = &_par_cleanup_thread_state[t]->list;
1808 assert(list->well_formed(), "Inv");
1809 HeapRegion* hd = list->hd();
1810 while (hd != NULL) {
1811 // Now finish up the other stuff.
1812 hd->rem_set()->clear();
1813 HeapRegion* next_hd = hd->next_from_unclean_list();
1814 (void)list->pop();
1815 guarantee(list->hd() == next_hd, "how not?");
1816 _g1h->put_region_on_unclean_list(hd);
1817 if (!hd->isHumongous()) {
1818 // Add this to the _free_regions count by 1.
1819 _g1h->finish_free_region_work(0, 0, 1, NULL);
1820 }
1821 hd = list->hd();
1822 guarantee(hd == next_hd, "how not?");
1823 }
1824 }
1825 }
1828 class G1CMIsAliveClosure: public BoolObjectClosure {
1829 G1CollectedHeap* _g1;
1830 public:
1831 G1CMIsAliveClosure(G1CollectedHeap* g1) :
1832 _g1(g1)
1833 {}
1835 void do_object(oop obj) {
1836 assert(false, "not to be invoked");
1837 }
1838 bool do_object_b(oop obj) {
1839 HeapWord* addr = (HeapWord*)obj;
1840 return addr != NULL &&
1841 (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj));
1842 }
1843 };
1845 class G1CMKeepAliveClosure: public OopClosure {
1846 G1CollectedHeap* _g1;
1847 ConcurrentMark* _cm;
1848 CMBitMap* _bitMap;
1849 public:
1850 G1CMKeepAliveClosure(G1CollectedHeap* g1, ConcurrentMark* cm,
1851 CMBitMap* bitMap) :
1852 _g1(g1), _cm(cm),
1853 _bitMap(bitMap) {}
1855 void do_oop(narrowOop* p) {
1856 guarantee(false, "NYI");
1857 }
1859 void do_oop(oop* p) {
1860 oop thisOop = *p;
1861 HeapWord* addr = (HeapWord*)thisOop;
1862 if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(thisOop)) {
1863 _bitMap->mark(addr);
1864 _cm->mark_stack_push(thisOop);
1865 }
1866 }
1867 };
1869 class G1CMDrainMarkingStackClosure: public VoidClosure {
1870 CMMarkStack* _markStack;
1871 CMBitMap* _bitMap;
1872 G1CMKeepAliveClosure* _oopClosure;
1873 public:
1874 G1CMDrainMarkingStackClosure(CMBitMap* bitMap, CMMarkStack* markStack,
1875 G1CMKeepAliveClosure* oopClosure) :
1876 _bitMap(bitMap),
1877 _markStack(markStack),
1878 _oopClosure(oopClosure)
1879 {}
1881 void do_void() {
1882 _markStack->drain((OopClosure*)_oopClosure, _bitMap, false);
1883 }
1884 };
1886 void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {
1887 ResourceMark rm;
1888 HandleMark hm;
1889 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1890 ReferenceProcessor* rp = g1h->ref_processor();
1892 // Process weak references.
1893 rp->setup_policy(clear_all_soft_refs);
1894 assert(_markStack.isEmpty(), "mark stack should be empty");
1896 G1CMIsAliveClosure g1IsAliveClosure (g1h);
1897 G1CMKeepAliveClosure g1KeepAliveClosure(g1h, this, nextMarkBitMap());
1898 G1CMDrainMarkingStackClosure
1899 g1DrainMarkingStackClosure(nextMarkBitMap(), &_markStack,
1900 &g1KeepAliveClosure);
1902 // XXXYYY Also: copy the parallel ref processing code from CMS.
1903 rp->process_discovered_references(&g1IsAliveClosure,
1904 &g1KeepAliveClosure,
1905 &g1DrainMarkingStackClosure,
1906 NULL);
1907 assert(_markStack.overflow() || _markStack.isEmpty(),
1908 "mark stack should be empty (unless it overflowed)");
1909 if (_markStack.overflow()) {
1910 set_has_overflown();
1911 }
1913 rp->enqueue_discovered_references();
1914 rp->verify_no_references_recorded();
1915 assert(!rp->discovery_enabled(), "should have been disabled");
1917 // Now clean up stale oops in SymbolTable and StringTable
1918 SymbolTable::unlink(&g1IsAliveClosure);
1919 StringTable::unlink(&g1IsAliveClosure);
1920 }
1922 void ConcurrentMark::swapMarkBitMaps() {
1923 CMBitMapRO* temp = _prevMarkBitMap;
1924 _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap;
1925 _nextMarkBitMap = (CMBitMap*) temp;
1926 }
1928 class CMRemarkTask: public AbstractGangTask {
1929 private:
1930 ConcurrentMark *_cm;
1932 public:
1933 void work(int worker_i) {
1934 // Since all available tasks are actually started, we should
1935 // only proceed if we're supposed to be actived.
1936 if ((size_t)worker_i < _cm->active_tasks()) {
1937 CMTask* task = _cm->task(worker_i);
1938 task->record_start_time();
1939 do {
1940 task->do_marking_step(1000000000.0 /* something very large */);
1941 } while (task->has_aborted() && !_cm->has_overflown());
1942 // If we overflow, then we do not want to restart. We instead
1943 // want to abort remark and do concurrent marking again.
1944 task->record_end_time();
1945 }
1946 }
1948 CMRemarkTask(ConcurrentMark* cm) :
1949 AbstractGangTask("Par Remark"), _cm(cm) { }
1950 };
1952 void ConcurrentMark::checkpointRootsFinalWork() {
1953 ResourceMark rm;
1954 HandleMark hm;
1955 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1957 g1h->ensure_parsability(false);
1959 if (ParallelGCThreads > 0) {
1960 g1h->change_strong_roots_parity();
1961 // this is remark, so we'll use up all available threads
1962 int active_workers = ParallelGCThreads;
1963 set_phase(active_workers, false);
1965 CMRemarkTask remarkTask(this);
1966 // We will start all available threads, even if we decide that the
1967 // active_workers will be fewer. The extra ones will just bail out
1968 // immediately.
1969 int n_workers = g1h->workers()->total_workers();
1970 g1h->set_par_threads(n_workers);
1971 g1h->workers()->run_task(&remarkTask);
1972 g1h->set_par_threads(0);
1974 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1975 guarantee( satb_mq_set.completed_buffers_num() == 0, "invariant" );
1976 } else {
1977 g1h->change_strong_roots_parity();
1978 // this is remark, so we'll use up all available threads
1979 int active_workers = 1;
1980 set_phase(active_workers, false);
1982 CMRemarkTask remarkTask(this);
1983 // We will start all available threads, even if we decide that the
1984 // active_workers will be fewer. The extra ones will just bail out
1985 // immediately.
1986 remarkTask.work(0);
1988 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1989 guarantee( satb_mq_set.completed_buffers_num() == 0, "invariant" );
1990 }
1992 print_stats();
1994 if (!restart_for_overflow())
1995 set_non_marking_state();
1997 #if VERIFY_OBJS_PROCESSED
1998 if (_scan_obj_cl.objs_processed != ThreadLocalObjQueue::objs_enqueued) {
1999 gclog_or_tty->print_cr("Processed = %d, enqueued = %d.",
2000 _scan_obj_cl.objs_processed,
2001 ThreadLocalObjQueue::objs_enqueued);
2002 guarantee(_scan_obj_cl.objs_processed ==
2003 ThreadLocalObjQueue::objs_enqueued,
2004 "Different number of objs processed and enqueued.");
2005 }
2006 #endif
2007 }
2009 class ReachablePrinterOopClosure: public OopClosure {
2010 private:
2011 G1CollectedHeap* _g1h;
2012 CMBitMapRO* _bitmap;
2013 outputStream* _out;
2015 public:
2016 ReachablePrinterOopClosure(CMBitMapRO* bitmap, outputStream* out) :
2017 _bitmap(bitmap), _g1h(G1CollectedHeap::heap()), _out(out) { }
2019 void do_oop(narrowOop* p) {
2020 guarantee(false, "NYI");
2021 }
2023 void do_oop(oop* p) {
2024 oop obj = *p;
2025 const char* str = NULL;
2026 const char* str2 = "";
2028 if (!_g1h->is_in_g1_reserved(obj))
2029 str = "outside G1 reserved";
2030 else {
2031 HeapRegion* hr = _g1h->heap_region_containing(obj);
2032 guarantee( hr != NULL, "invariant" );
2033 if (hr->obj_allocated_since_prev_marking(obj)) {
2034 str = "over TAMS";
2035 if (_bitmap->isMarked((HeapWord*) obj))
2036 str2 = " AND MARKED";
2037 } else if (_bitmap->isMarked((HeapWord*) obj))
2038 str = "marked";
2039 else
2040 str = "#### NOT MARKED ####";
2041 }
2043 _out->print_cr(" "PTR_FORMAT" contains "PTR_FORMAT" %s%s",
2044 p, (void*) obj, str, str2);
2045 }
2046 };
2048 class ReachablePrinterClosure: public BitMapClosure {
2049 private:
2050 CMBitMapRO* _bitmap;
2051 outputStream* _out;
2053 public:
2054 ReachablePrinterClosure(CMBitMapRO* bitmap, outputStream* out) :
2055 _bitmap(bitmap), _out(out) { }
2057 bool do_bit(size_t offset) {
2058 HeapWord* addr = _bitmap->offsetToHeapWord(offset);
2059 ReachablePrinterOopClosure oopCl(_bitmap, _out);
2061 _out->print_cr(" obj "PTR_FORMAT", offset %10d (marked)", addr, offset);
2062 oop(addr)->oop_iterate(&oopCl);
2063 _out->print_cr("");
2065 return true;
2066 }
2067 };
2069 class ObjInRegionReachablePrinterClosure : public ObjectClosure {
2070 private:
2071 CMBitMapRO* _bitmap;
2072 outputStream* _out;
2074 public:
2075 void do_object(oop o) {
2076 ReachablePrinterOopClosure oopCl(_bitmap, _out);
2078 _out->print_cr(" obj "PTR_FORMAT" (over TAMS)", (void*) o);
2079 o->oop_iterate(&oopCl);
2080 _out->print_cr("");
2081 }
2083 ObjInRegionReachablePrinterClosure(CMBitMapRO* bitmap, outputStream* out) :
2084 _bitmap(bitmap), _out(out) { }
2085 };
2087 class RegionReachablePrinterClosure : public HeapRegionClosure {
2088 private:
2089 CMBitMapRO* _bitmap;
2090 outputStream* _out;
2092 public:
2093 bool doHeapRegion(HeapRegion* hr) {
2094 HeapWord* b = hr->bottom();
2095 HeapWord* e = hr->end();
2096 HeapWord* t = hr->top();
2097 HeapWord* p = hr->prev_top_at_mark_start();
2098 _out->print_cr("** ["PTR_FORMAT", "PTR_FORMAT"] top: "PTR_FORMAT" "
2099 "PTAMS: "PTR_FORMAT, b, e, t, p);
2100 _out->print_cr("");
2102 ObjInRegionReachablePrinterClosure ocl(_bitmap, _out);
2103 hr->object_iterate_mem_careful(MemRegion(p, t), &ocl);
2105 return false;
2106 }
2108 RegionReachablePrinterClosure(CMBitMapRO* bitmap,
2109 outputStream* out) :
2110 _bitmap(bitmap), _out(out) { }
2111 };
2113 void ConcurrentMark::print_prev_bitmap_reachable() {
2114 outputStream* out = gclog_or_tty;
2116 #if SEND_HEAP_DUMP_TO_FILE
2117 guarantee(heap_dump_file == NULL, "Protocol");
2118 char fn_buf[100];
2119 sprintf(fn_buf, "/tmp/dump.txt.%d", os::current_process_id());
2120 heap_dump_file = fopen(fn_buf, "w");
2121 fileStream fstream(heap_dump_file);
2122 out = &fstream;
2123 #endif // SEND_HEAP_DUMP_TO_FILE
2125 RegionReachablePrinterClosure rcl(_prevMarkBitMap, out);
2126 out->print_cr("--- ITERATING OVER REGIONS WITH PTAMS < TOP");
2127 _g1h->heap_region_iterate(&rcl);
2128 out->print_cr("");
2130 ReachablePrinterClosure cl(_prevMarkBitMap, out);
2131 out->print_cr("--- REACHABLE OBJECTS ON THE BITMAP");
2132 _prevMarkBitMap->iterate(&cl);
2133 out->print_cr("");
2135 #if SEND_HEAP_DUMP_TO_FILE
2136 fclose(heap_dump_file);
2137 heap_dump_file = NULL;
2138 #endif // SEND_HEAP_DUMP_TO_FILE
2139 }
2141 // This note is for drainAllSATBBuffers and the code in between.
2142 // In the future we could reuse a task to do this work during an
2143 // evacuation pause (since now tasks are not active and can be claimed
2144 // during an evacuation pause). This was a late change to the code and
2145 // is currently not being taken advantage of.
2147 class CMGlobalObjectClosure : public ObjectClosure {
2148 private:
2149 ConcurrentMark* _cm;
2151 public:
2152 void do_object(oop obj) {
2153 _cm->deal_with_reference(obj);
2154 }
2156 CMGlobalObjectClosure(ConcurrentMark* cm) : _cm(cm) { }
2157 };
2159 void ConcurrentMark::deal_with_reference(oop obj) {
2160 if (verbose_high())
2161 gclog_or_tty->print_cr("[global] we're dealing with reference "PTR_FORMAT,
2162 (void*) obj);
2165 HeapWord* objAddr = (HeapWord*) obj;
2166 if (_g1h->is_in_g1_reserved(objAddr)) {
2167 tmp_guarantee_CM( obj != NULL, "is_in_g1_reserved should ensure this" );
2168 HeapRegion* hr = _g1h->heap_region_containing(obj);
2169 if (_g1h->is_obj_ill(obj, hr)) {
2170 if (verbose_high())
2171 gclog_or_tty->print_cr("[global] "PTR_FORMAT" is not considered "
2172 "marked", (void*) obj);
2174 // we need to mark it first
2175 if (_nextMarkBitMap->parMark(objAddr)) {
2176 // No OrderAccess:store_load() is needed. It is implicit in the
2177 // CAS done in parMark(objAddr) above
2178 HeapWord* finger = _finger;
2179 if (objAddr < finger) {
2180 if (verbose_high())
2181 gclog_or_tty->print_cr("[global] below the global finger "
2182 "("PTR_FORMAT"), pushing it", finger);
2183 if (!mark_stack_push(obj)) {
2184 if (verbose_low())
2185 gclog_or_tty->print_cr("[global] global stack overflow during "
2186 "deal_with_reference");
2187 }
2188 }
2189 }
2190 }
2191 }
2192 }
2194 void ConcurrentMark::drainAllSATBBuffers() {
2195 CMGlobalObjectClosure oc(this);
2196 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2197 satb_mq_set.set_closure(&oc);
2199 while (satb_mq_set.apply_closure_to_completed_buffer()) {
2200 if (verbose_medium())
2201 gclog_or_tty->print_cr("[global] processed an SATB buffer");
2202 }
2204 // no need to check whether we should do this, as this is only
2205 // called during an evacuation pause
2206 satb_mq_set.iterate_closure_all_threads();
2208 satb_mq_set.set_closure(NULL);
2209 guarantee( satb_mq_set.completed_buffers_num() == 0, "invariant" );
2210 }
2212 void ConcurrentMark::markPrev(oop p) {
2213 // Note we are overriding the read-only view of the prev map here, via
2214 // the cast.
2215 ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*)p);
2216 }
2218 void ConcurrentMark::clear(oop p) {
2219 assert(p != NULL && p->is_oop(), "expected an oop");
2220 HeapWord* addr = (HeapWord*)p;
2221 assert(addr >= _nextMarkBitMap->startWord() ||
2222 addr < _nextMarkBitMap->endWord(), "in a region");
2224 _nextMarkBitMap->clear(addr);
2225 }
2227 void ConcurrentMark::clearRangeBothMaps(MemRegion mr) {
2228 // Note we are overriding the read-only view of the prev map here, via
2229 // the cast.
2230 ((CMBitMap*)_prevMarkBitMap)->clearRange(mr);
2231 _nextMarkBitMap->clearRange(mr);
2232 }
2234 HeapRegion*
2235 ConcurrentMark::claim_region(int task_num) {
2236 // "checkpoint" the finger
2237 HeapWord* finger = _finger;
2239 // _heap_end will not change underneath our feet; it only changes at
2240 // yield points.
2241 while (finger < _heap_end) {
2242 tmp_guarantee_CM( _g1h->is_in_g1_reserved(finger), "invariant" );
2244 // is the gap between reading the finger and doing the CAS too long?
2246 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
2247 HeapWord* bottom = curr_region->bottom();
2248 HeapWord* end = curr_region->end();
2249 HeapWord* limit = curr_region->next_top_at_mark_start();
2251 if (verbose_low())
2252 gclog_or_tty->print_cr("[%d] curr_region = "PTR_FORMAT" "
2253 "["PTR_FORMAT", "PTR_FORMAT"), "
2254 "limit = "PTR_FORMAT,
2255 task_num, curr_region, bottom, end, limit);
2257 HeapWord* res =
2258 (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger);
2259 if (res == finger) {
2260 // we succeeded
2262 // notice that _finger == end cannot be guaranteed here since,
2263 // someone else might have moved the finger even further
2264 guarantee( _finger >= end, "the finger should have moved forward" );
2266 if (verbose_low())
2267 gclog_or_tty->print_cr("[%d] we were successful with region = "
2268 PTR_FORMAT, task_num, curr_region);
2270 if (limit > bottom) {
2271 if (verbose_low())
2272 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is not empty, "
2273 "returning it ", task_num, curr_region);
2274 return curr_region;
2275 } else {
2276 tmp_guarantee_CM( limit == bottom,
2277 "the region limit should be at bottom" );
2278 if (verbose_low())
2279 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is empty, "
2280 "returning NULL", task_num, curr_region);
2281 // we return NULL and the caller should try calling
2282 // claim_region() again.
2283 return NULL;
2284 }
2285 } else {
2286 guarantee( _finger > finger, "the finger should have moved forward" );
2287 if (verbose_low())
2288 gclog_or_tty->print_cr("[%d] somebody else moved the finger, "
2289 "global finger = "PTR_FORMAT", "
2290 "our finger = "PTR_FORMAT,
2291 task_num, _finger, finger);
2293 // read it again
2294 finger = _finger;
2295 }
2296 }
2298 return NULL;
2299 }
2301 void ConcurrentMark::oops_do(OopClosure* cl) {
2302 if (_markStack.size() > 0 && verbose_low())
2303 gclog_or_tty->print_cr("[global] scanning the global marking stack, "
2304 "size = %d", _markStack.size());
2305 // we first iterate over the contents of the mark stack...
2306 _markStack.oops_do(cl);
2308 for (int i = 0; i < (int)_max_task_num; ++i) {
2309 OopTaskQueue* queue = _task_queues->queue((int)i);
2311 if (queue->size() > 0 && verbose_low())
2312 gclog_or_tty->print_cr("[global] scanning task queue of task %d, "
2313 "size = %d", i, queue->size());
2315 // ...then over the contents of the all the task queues.
2316 queue->oops_do(cl);
2317 }
2319 // finally, invalidate any entries that in the region stack that
2320 // point into the collection set
2321 if (_regionStack.invalidate_entries_into_cset()) {
2322 // otherwise, any gray objects copied during the evacuation pause
2323 // might not be visited.
2324 guarantee( _should_gray_objects, "invariant" );
2325 }
2326 }
2328 void ConcurrentMark::clear_marking_state() {
2329 _markStack.setEmpty();
2330 _markStack.clear_overflow();
2331 _regionStack.setEmpty();
2332 _regionStack.clear_overflow();
2333 clear_has_overflown();
2334 _finger = _heap_start;
2336 for (int i = 0; i < (int)_max_task_num; ++i) {
2337 OopTaskQueue* queue = _task_queues->queue(i);
2338 queue->set_empty();
2339 }
2340 }
2342 void ConcurrentMark::print_stats() {
2343 if (verbose_stats()) {
2344 gclog_or_tty->print_cr("---------------------------------------------------------------------");
2345 for (size_t i = 0; i < _active_tasks; ++i) {
2346 _tasks[i]->print_stats();
2347 gclog_or_tty->print_cr("---------------------------------------------------------------------");
2348 }
2349 }
2350 }
2352 class CSMarkOopClosure: public OopClosure {
2353 friend class CSMarkBitMapClosure;
2355 G1CollectedHeap* _g1h;
2356 CMBitMap* _bm;
2357 ConcurrentMark* _cm;
2358 oop* _ms;
2359 jint* _array_ind_stack;
2360 int _ms_size;
2361 int _ms_ind;
2362 int _array_increment;
2364 bool push(oop obj, int arr_ind = 0) {
2365 if (_ms_ind == _ms_size) {
2366 gclog_or_tty->print_cr("Mark stack is full.");
2367 return false;
2368 }
2369 _ms[_ms_ind] = obj;
2370 if (obj->is_objArray()) _array_ind_stack[_ms_ind] = arr_ind;
2371 _ms_ind++;
2372 return true;
2373 }
2375 oop pop() {
2376 if (_ms_ind == 0) return NULL;
2377 else {
2378 _ms_ind--;
2379 return _ms[_ms_ind];
2380 }
2381 }
2383 bool drain() {
2384 while (_ms_ind > 0) {
2385 oop obj = pop();
2386 assert(obj != NULL, "Since index was non-zero.");
2387 if (obj->is_objArray()) {
2388 jint arr_ind = _array_ind_stack[_ms_ind];
2389 objArrayOop aobj = objArrayOop(obj);
2390 jint len = aobj->length();
2391 jint next_arr_ind = arr_ind + _array_increment;
2392 if (next_arr_ind < len) {
2393 push(obj, next_arr_ind);
2394 }
2395 // Now process this portion of this one.
2396 int lim = MIN2(next_arr_ind, len);
2397 assert(!UseCompressedOops, "This needs to be fixed");
2398 for (int j = arr_ind; j < lim; j++) {
2399 do_oop(aobj->obj_at_addr<oop>(j));
2400 }
2402 } else {
2403 obj->oop_iterate(this);
2404 }
2405 if (abort()) return false;
2406 }
2407 return true;
2408 }
2410 public:
2411 CSMarkOopClosure(ConcurrentMark* cm, int ms_size) :
2412 _g1h(G1CollectedHeap::heap()),
2413 _cm(cm),
2414 _bm(cm->nextMarkBitMap()),
2415 _ms_size(ms_size), _ms_ind(0),
2416 _ms(NEW_C_HEAP_ARRAY(oop, ms_size)),
2417 _array_ind_stack(NEW_C_HEAP_ARRAY(jint, ms_size)),
2418 _array_increment(MAX2(ms_size/8, 16))
2419 {}
2421 ~CSMarkOopClosure() {
2422 FREE_C_HEAP_ARRAY(oop, _ms);
2423 FREE_C_HEAP_ARRAY(jint, _array_ind_stack);
2424 }
2426 void do_oop(narrowOop* p) {
2427 guarantee(false, "NYI");
2428 }
2430 void do_oop(oop* p) {
2431 oop obj = *p;
2432 if (obj == NULL) return;
2433 if (obj->is_forwarded()) {
2434 // If the object has already been forwarded, we have to make sure
2435 // that it's marked. So follow the forwarding pointer. Note that
2436 // this does the right thing for self-forwarding pointers in the
2437 // evacuation failure case.
2438 obj = obj->forwardee();
2439 }
2440 HeapRegion* hr = _g1h->heap_region_containing(obj);
2441 if (hr != NULL) {
2442 if (hr->in_collection_set()) {
2443 if (_g1h->is_obj_ill(obj)) {
2444 _bm->mark((HeapWord*)obj);
2445 if (!push(obj)) {
2446 gclog_or_tty->print_cr("Setting abort in CSMarkOopClosure because push failed.");
2447 set_abort();
2448 }
2449 }
2450 } else {
2451 // Outside the collection set; we need to gray it
2452 _cm->deal_with_reference(obj);
2453 }
2454 }
2455 }
2456 };
2458 class CSMarkBitMapClosure: public BitMapClosure {
2459 G1CollectedHeap* _g1h;
2460 CMBitMap* _bitMap;
2461 ConcurrentMark* _cm;
2462 CSMarkOopClosure _oop_cl;
2463 public:
2464 CSMarkBitMapClosure(ConcurrentMark* cm, int ms_size) :
2465 _g1h(G1CollectedHeap::heap()),
2466 _bitMap(cm->nextMarkBitMap()),
2467 _oop_cl(cm, ms_size)
2468 {}
2470 ~CSMarkBitMapClosure() {}
2472 bool do_bit(size_t offset) {
2473 // convert offset into a HeapWord*
2474 HeapWord* addr = _bitMap->offsetToHeapWord(offset);
2475 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
2476 "address out of range");
2477 assert(_bitMap->isMarked(addr), "tautology");
2478 oop obj = oop(addr);
2479 if (!obj->is_forwarded()) {
2480 if (!_oop_cl.push(obj)) return false;
2481 if (!_oop_cl.drain()) return false;
2482 }
2483 // Otherwise...
2484 return true;
2485 }
2486 };
2489 class CompleteMarkingInCSHRClosure: public HeapRegionClosure {
2490 CMBitMap* _bm;
2491 CSMarkBitMapClosure _bit_cl;
2492 enum SomePrivateConstants {
2493 MSSize = 1000
2494 };
2495 bool _completed;
2496 public:
2497 CompleteMarkingInCSHRClosure(ConcurrentMark* cm) :
2498 _bm(cm->nextMarkBitMap()),
2499 _bit_cl(cm, MSSize),
2500 _completed(true)
2501 {}
2503 ~CompleteMarkingInCSHRClosure() {}
2505 bool doHeapRegion(HeapRegion* r) {
2506 if (!r->evacuation_failed()) {
2507 MemRegion mr = MemRegion(r->bottom(), r->next_top_at_mark_start());
2508 if (!mr.is_empty()) {
2509 if (!_bm->iterate(&_bit_cl, mr)) {
2510 _completed = false;
2511 return true;
2512 }
2513 }
2514 }
2515 return false;
2516 }
2518 bool completed() { return _completed; }
2519 };
2521 class ClearMarksInHRClosure: public HeapRegionClosure {
2522 CMBitMap* _bm;
2523 public:
2524 ClearMarksInHRClosure(CMBitMap* bm): _bm(bm) { }
2526 bool doHeapRegion(HeapRegion* r) {
2527 if (!r->used_region().is_empty() && !r->evacuation_failed()) {
2528 MemRegion usedMR = r->used_region();
2529 _bm->clearRange(r->used_region());
2530 }
2531 return false;
2532 }
2533 };
2535 void ConcurrentMark::complete_marking_in_collection_set() {
2536 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2538 if (!g1h->mark_in_progress()) {
2539 g1h->g1_policy()->record_mark_closure_time(0.0);
2540 return;
2541 }
2543 int i = 1;
2544 double start = os::elapsedTime();
2545 while (true) {
2546 i++;
2547 CompleteMarkingInCSHRClosure cmplt(this);
2548 g1h->collection_set_iterate(&cmplt);
2549 if (cmplt.completed()) break;
2550 }
2551 double end_time = os::elapsedTime();
2552 double elapsed_time_ms = (end_time - start) * 1000.0;
2553 g1h->g1_policy()->record_mark_closure_time(elapsed_time_ms);
2554 if (PrintGCDetails) {
2555 gclog_or_tty->print_cr("Mark closure took %5.2f ms.", elapsed_time_ms);
2556 }
2558 ClearMarksInHRClosure clr(nextMarkBitMap());
2559 g1h->collection_set_iterate(&clr);
2560 }
2562 // The next two methods deal with the following optimisation. Some
2563 // objects are gray by being marked and located above the finger. If
2564 // they are copied, during an evacuation pause, below the finger then
2565 // the need to be pushed on the stack. The observation is that, if
2566 // there are no regions in the collection set located above the
2567 // finger, then the above cannot happen, hence we do not need to
2568 // explicitly gray any objects when copying them to below the
2569 // finger. The global stack will be scanned to ensure that, if it
2570 // points to objects being copied, it will update their
2571 // location. There is a tricky situation with the gray objects in
2572 // region stack that are being coped, however. See the comment in
2573 // newCSet().
2575 void ConcurrentMark::newCSet() {
2576 if (!concurrent_marking_in_progress())
2577 // nothing to do if marking is not in progress
2578 return;
2580 // find what the lowest finger is among the global and local fingers
2581 _min_finger = _finger;
2582 for (int i = 0; i < (int)_max_task_num; ++i) {
2583 CMTask* task = _tasks[i];
2584 HeapWord* task_finger = task->finger();
2585 if (task_finger != NULL && task_finger < _min_finger)
2586 _min_finger = task_finger;
2587 }
2589 _should_gray_objects = false;
2591 // This fixes a very subtle and fustrating bug. It might be the case
2592 // that, during en evacuation pause, heap regions that contain
2593 // objects that are gray (by being in regions contained in the
2594 // region stack) are included in the collection set. Since such gray
2595 // objects will be moved, and because it's not easy to redirect
2596 // region stack entries to point to a new location (because objects
2597 // in one region might be scattered to multiple regions after they
2598 // are copied), one option is to ensure that all marked objects
2599 // copied during a pause are pushed on the stack. Notice, however,
2600 // that this problem can only happen when the region stack is not
2601 // empty during an evacuation pause. So, we make the fix a bit less
2602 // conservative and ensure that regions are pushed on the stack,
2603 // irrespective whether all collection set regions are below the
2604 // finger, if the region stack is not empty. This is expected to be
2605 // a rare case, so I don't think it's necessary to be smarted about it.
2606 if (!region_stack_empty())
2607 _should_gray_objects = true;
2608 }
2610 void ConcurrentMark::registerCSetRegion(HeapRegion* hr) {
2611 if (!concurrent_marking_in_progress())
2612 return;
2614 HeapWord* region_end = hr->end();
2615 if (region_end > _min_finger)
2616 _should_gray_objects = true;
2617 }
2619 void ConcurrentMark::disable_co_trackers() {
2620 if (has_aborted()) {
2621 if (_cleanup_co_tracker.enabled())
2622 _cleanup_co_tracker.disable();
2623 for (int i = 0; i < (int)_max_task_num; ++i) {
2624 CMTask* task = _tasks[i];
2625 if (task->co_tracker_enabled())
2626 task->disable_co_tracker();
2627 }
2628 } else {
2629 guarantee( !_cleanup_co_tracker.enabled(), "invariant" );
2630 for (int i = 0; i < (int)_max_task_num; ++i) {
2631 CMTask* task = _tasks[i];
2632 guarantee( !task->co_tracker_enabled(), "invariant" );
2633 }
2634 }
2635 }
2637 // abandon current marking iteration due to a Full GC
2638 void ConcurrentMark::abort() {
2639 // If we're not marking, nothing to do.
2640 if (!G1ConcMark) return;
2642 // Clear all marks to force marking thread to do nothing
2643 _nextMarkBitMap->clearAll();
2644 // Empty mark stack
2645 clear_marking_state();
2646 for (int i = 0; i < (int)_max_task_num; ++i)
2647 _tasks[i]->clear_region_fields();
2648 _has_aborted = true;
2650 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2651 satb_mq_set.abandon_partial_marking();
2652 satb_mq_set.set_active_all_threads(false);
2653 }
2655 static void print_ms_time_info(const char* prefix, const char* name,
2656 NumberSeq& ns) {
2657 gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2658 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2659 if (ns.num() > 0) {
2660 gclog_or_tty->print_cr("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2661 prefix, ns.sd(), ns.maximum());
2662 }
2663 }
2665 void ConcurrentMark::print_summary_info() {
2666 gclog_or_tty->print_cr(" Concurrent marking:");
2667 print_ms_time_info(" ", "init marks", _init_times);
2668 print_ms_time_info(" ", "remarks", _remark_times);
2669 {
2670 print_ms_time_info(" ", "final marks", _remark_mark_times);
2671 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2673 }
2674 print_ms_time_info(" ", "cleanups", _cleanup_times);
2675 gclog_or_tty->print_cr(" Final counting total time = %8.2f s (avg = %8.2f ms).",
2676 _total_counting_time,
2677 (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 /
2678 (double)_cleanup_times.num()
2679 : 0.0));
2680 if (G1ScrubRemSets) {
2681 gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
2682 _total_rs_scrub_time,
2683 (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 /
2684 (double)_cleanup_times.num()
2685 : 0.0));
2686 }
2687 gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.",
2688 (_init_times.sum() + _remark_times.sum() +
2689 _cleanup_times.sum())/1000.0);
2690 gclog_or_tty->print_cr(" Total concurrent time = %8.2f s "
2691 "(%8.2f s marking, %8.2f s counting).",
2692 cmThread()->vtime_accum(),
2693 cmThread()->vtime_mark_accum(),
2694 cmThread()->vtime_count_accum());
2695 }
2697 // Closures
2698 // XXX: there seems to be a lot of code duplication here;
2699 // should refactor and consolidate the shared code.
2701 // This closure is used to mark refs into the CMS generation in
2702 // the CMS bit map. Called at the first checkpoint.
2704 // We take a break if someone is trying to stop the world.
2705 bool ConcurrentMark::do_yield_check(int worker_i) {
2706 if (should_yield()) {
2707 if (worker_i == 0)
2708 _g1h->g1_policy()->record_concurrent_pause();
2709 cmThread()->yield();
2710 if (worker_i == 0)
2711 _g1h->g1_policy()->record_concurrent_pause_end();
2712 return true;
2713 } else {
2714 return false;
2715 }
2716 }
2718 bool ConcurrentMark::should_yield() {
2719 return cmThread()->should_yield();
2720 }
2722 bool ConcurrentMark::containing_card_is_marked(void* p) {
2723 size_t offset = pointer_delta(p, _g1h->reserved_region().start(), 1);
2724 return _card_bm.at(offset >> CardTableModRefBS::card_shift);
2725 }
2727 bool ConcurrentMark::containing_cards_are_marked(void* start,
2728 void* last) {
2729 return
2730 containing_card_is_marked(start) &&
2731 containing_card_is_marked(last);
2732 }
2734 #ifndef PRODUCT
2735 // for debugging purposes
2736 void ConcurrentMark::print_finger() {
2737 gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT,
2738 _heap_start, _heap_end, _finger);
2739 for (int i = 0; i < (int) _max_task_num; ++i) {
2740 gclog_or_tty->print(" %d: "PTR_FORMAT, i, _tasks[i]->finger());
2741 }
2742 gclog_or_tty->print_cr("");
2743 }
2744 #endif
2746 // Closure for iteration over bitmaps
2747 class CMBitMapClosure : public BitMapClosure {
2748 private:
2749 // the bitmap that is being iterated over
2750 CMBitMap* _nextMarkBitMap;
2751 ConcurrentMark* _cm;
2752 CMTask* _task;
2753 // true if we're scanning a heap region claimed by the task (so that
2754 // we move the finger along), false if we're not, i.e. currently when
2755 // scanning a heap region popped from the region stack (so that we
2756 // do not move the task finger along; it'd be a mistake if we did so).
2757 bool _scanning_heap_region;
2759 public:
2760 CMBitMapClosure(CMTask *task,
2761 ConcurrentMark* cm,
2762 CMBitMap* nextMarkBitMap)
2763 : _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { }
2765 void set_scanning_heap_region(bool scanning_heap_region) {
2766 _scanning_heap_region = scanning_heap_region;
2767 }
2769 bool do_bit(size_t offset) {
2770 HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset);
2771 tmp_guarantee_CM( _nextMarkBitMap->isMarked(addr), "invariant" );
2772 tmp_guarantee_CM( addr < _cm->finger(), "invariant" );
2774 if (_scanning_heap_region) {
2775 statsOnly( _task->increase_objs_found_on_bitmap() );
2776 tmp_guarantee_CM( addr >= _task->finger(), "invariant" );
2777 // We move that task's local finger along.
2778 _task->move_finger_to(addr);
2779 } else {
2780 // We move the task's region finger along.
2781 _task->move_region_finger_to(addr);
2782 }
2784 _task->scan_object(oop(addr));
2785 // we only partially drain the local queue and global stack
2786 _task->drain_local_queue(true);
2787 _task->drain_global_stack(true);
2789 // if the has_aborted flag has been raised, we need to bail out of
2790 // the iteration
2791 return !_task->has_aborted();
2792 }
2793 };
2795 // Closure for iterating over objects, currently only used for
2796 // processing SATB buffers.
2797 class CMObjectClosure : public ObjectClosure {
2798 private:
2799 CMTask* _task;
2801 public:
2802 void do_object(oop obj) {
2803 _task->deal_with_reference(obj);
2804 }
2806 CMObjectClosure(CMTask* task) : _task(task) { }
2807 };
2809 // Closure for iterating over object fields
2810 class CMOopClosure : public OopClosure {
2811 private:
2812 G1CollectedHeap* _g1h;
2813 ConcurrentMark* _cm;
2814 CMTask* _task;
2816 public:
2817 void do_oop(narrowOop* p) {
2818 guarantee(false, "NYI");
2819 }
2821 void do_oop(oop* p) {
2822 tmp_guarantee_CM( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant" );
2824 oop obj = *p;
2825 if (_cm->verbose_high())
2826 gclog_or_tty->print_cr("[%d] we're looking at location "
2827 "*"PTR_FORMAT" = "PTR_FORMAT,
2828 _task->task_id(), p, (void*) obj);
2829 _task->deal_with_reference(obj);
2830 }
2832 CMOopClosure(G1CollectedHeap* g1h,
2833 ConcurrentMark* cm,
2834 CMTask* task)
2835 : _g1h(g1h), _cm(cm), _task(task) { }
2836 };
2838 void CMTask::setup_for_region(HeapRegion* hr) {
2839 tmp_guarantee_CM( hr != NULL && !hr->continuesHumongous(),
2840 "claim_region() should have filtered out continues humongous regions" );
2842 if (_cm->verbose_low())
2843 gclog_or_tty->print_cr("[%d] setting up for region "PTR_FORMAT,
2844 _task_id, hr);
2846 _curr_region = hr;
2847 _finger = hr->bottom();
2848 update_region_limit();
2849 }
2851 void CMTask::update_region_limit() {
2852 HeapRegion* hr = _curr_region;
2853 HeapWord* bottom = hr->bottom();
2854 HeapWord* limit = hr->next_top_at_mark_start();
2856 if (limit == bottom) {
2857 if (_cm->verbose_low())
2858 gclog_or_tty->print_cr("[%d] found an empty region "
2859 "["PTR_FORMAT", "PTR_FORMAT")",
2860 _task_id, bottom, limit);
2861 // The region was collected underneath our feet.
2862 // We set the finger to bottom to ensure that the bitmap
2863 // iteration that will follow this will not do anything.
2864 // (this is not a condition that holds when we set the region up,
2865 // as the region is not supposed to be empty in the first place)
2866 _finger = bottom;
2867 } else if (limit >= _region_limit) {
2868 tmp_guarantee_CM( limit >= _finger, "peace of mind" );
2869 } else {
2870 tmp_guarantee_CM( limit < _region_limit, "only way to get here" );
2871 // This can happen under some pretty unusual circumstances. An
2872 // evacuation pause empties the region underneath our feet (NTAMS
2873 // at bottom). We then do some allocation in the region (NTAMS
2874 // stays at bottom), followed by the region being used as a GC
2875 // alloc region (NTAMS will move to top() and the objects
2876 // originally below it will be grayed). All objects now marked in
2877 // the region are explicitly grayed, if below the global finger,
2878 // and we do not need in fact to scan anything else. So, we simply
2879 // set _finger to be limit to ensure that the bitmap iteration
2880 // doesn't do anything.
2881 _finger = limit;
2882 }
2884 _region_limit = limit;
2885 }
2887 void CMTask::giveup_current_region() {
2888 tmp_guarantee_CM( _curr_region != NULL, "invariant" );
2889 if (_cm->verbose_low())
2890 gclog_or_tty->print_cr("[%d] giving up region "PTR_FORMAT,
2891 _task_id, _curr_region);
2892 clear_region_fields();
2893 }
2895 void CMTask::clear_region_fields() {
2896 // Values for these three fields that indicate that we're not
2897 // holding on to a region.
2898 _curr_region = NULL;
2899 _finger = NULL;
2900 _region_limit = NULL;
2902 _region_finger = NULL;
2903 }
2905 void CMTask::reset(CMBitMap* nextMarkBitMap) {
2906 guarantee( nextMarkBitMap != NULL, "invariant" );
2908 if (_cm->verbose_low())
2909 gclog_or_tty->print_cr("[%d] resetting", _task_id);
2911 _nextMarkBitMap = nextMarkBitMap;
2912 clear_region_fields();
2914 _calls = 0;
2915 _elapsed_time_ms = 0.0;
2916 _termination_time_ms = 0.0;
2917 _termination_start_time_ms = 0.0;
2919 #if _MARKING_STATS_
2920 _local_pushes = 0;
2921 _local_pops = 0;
2922 _local_max_size = 0;
2923 _objs_scanned = 0;
2924 _global_pushes = 0;
2925 _global_pops = 0;
2926 _global_max_size = 0;
2927 _global_transfers_to = 0;
2928 _global_transfers_from = 0;
2929 _region_stack_pops = 0;
2930 _regions_claimed = 0;
2931 _objs_found_on_bitmap = 0;
2932 _satb_buffers_processed = 0;
2933 _steal_attempts = 0;
2934 _steals = 0;
2935 _aborted = 0;
2936 _aborted_overflow = 0;
2937 _aborted_cm_aborted = 0;
2938 _aborted_yield = 0;
2939 _aborted_timed_out = 0;
2940 _aborted_satb = 0;
2941 _aborted_termination = 0;
2942 #endif // _MARKING_STATS_
2943 }
2945 bool CMTask::should_exit_termination() {
2946 regular_clock_call();
2947 // This is called when we are in the termination protocol. We should
2948 // quit if, for some reason, this task wants to abort or the global
2949 // stack is not empty (this means that we can get work from it).
2950 return !_cm->mark_stack_empty() || has_aborted();
2951 }
2953 // This determines whether the method below will check both the local
2954 // and global fingers when determining whether to push on the stack a
2955 // gray object (value 1) or whether it will only check the global one
2956 // (value 0). The tradeoffs are that the former will be a bit more
2957 // accurate and possibly push less on the stack, but it might also be
2958 // a little bit slower.
2960 #define _CHECK_BOTH_FINGERS_ 1
2962 void CMTask::deal_with_reference(oop obj) {
2963 if (_cm->verbose_high())
2964 gclog_or_tty->print_cr("[%d] we're dealing with reference = "PTR_FORMAT,
2965 _task_id, (void*) obj);
2967 ++_refs_reached;
2969 HeapWord* objAddr = (HeapWord*) obj;
2970 if (_g1h->is_in_g1_reserved(objAddr)) {
2971 tmp_guarantee_CM( obj != NULL, "is_in_g1_reserved should ensure this" );
2972 HeapRegion* hr = _g1h->heap_region_containing(obj);
2973 if (_g1h->is_obj_ill(obj, hr)) {
2974 if (_cm->verbose_high())
2975 gclog_or_tty->print_cr("[%d] "PTR_FORMAT" is not considered marked",
2976 _task_id, (void*) obj);
2978 // we need to mark it first
2979 if (_nextMarkBitMap->parMark(objAddr)) {
2980 // No OrderAccess:store_load() is needed. It is implicit in the
2981 // CAS done in parMark(objAddr) above
2982 HeapWord* global_finger = _cm->finger();
2984 #if _CHECK_BOTH_FINGERS_
2985 // we will check both the local and global fingers
2987 if (_finger != NULL && objAddr < _finger) {
2988 if (_cm->verbose_high())
2989 gclog_or_tty->print_cr("[%d] below the local finger ("PTR_FORMAT"), "
2990 "pushing it", _task_id, _finger);
2991 push(obj);
2992 } else if (_curr_region != NULL && objAddr < _region_limit) {
2993 // do nothing
2994 } else if (objAddr < global_finger) {
2995 // Notice that the global finger might be moving forward
2996 // concurrently. This is not a problem. In the worst case, we
2997 // mark the object while it is above the global finger and, by
2998 // the time we read the global finger, it has moved forward
2999 // passed this object. In this case, the object will probably
3000 // be visited when a task is scanning the region and will also
3001 // be pushed on the stack. So, some duplicate work, but no
3002 // correctness problems.
3004 if (_cm->verbose_high())
3005 gclog_or_tty->print_cr("[%d] below the global finger "
3006 "("PTR_FORMAT"), pushing it",
3007 _task_id, global_finger);
3008 push(obj);
3009 } else {
3010 // do nothing
3011 }
3012 #else // _CHECK_BOTH_FINGERS_
3013 // we will only check the global finger
3015 if (objAddr < global_finger) {
3016 // see long comment above
3018 if (_cm->verbose_high())
3019 gclog_or_tty->print_cr("[%d] below the global finger "
3020 "("PTR_FORMAT"), pushing it",
3021 _task_id, global_finger);
3022 push(obj);
3023 }
3024 #endif // _CHECK_BOTH_FINGERS_
3025 }
3026 }
3027 }
3028 }
3030 void CMTask::push(oop obj) {
3031 HeapWord* objAddr = (HeapWord*) obj;
3032 tmp_guarantee_CM( _g1h->is_in_g1_reserved(objAddr), "invariant" );
3033 tmp_guarantee_CM( !_g1h->is_obj_ill(obj), "invariant" );
3034 tmp_guarantee_CM( _nextMarkBitMap->isMarked(objAddr), "invariant" );
3036 if (_cm->verbose_high())
3037 gclog_or_tty->print_cr("[%d] pushing "PTR_FORMAT, _task_id, (void*) obj);
3039 if (!_task_queue->push(obj)) {
3040 // The local task queue looks full. We need to push some entries
3041 // to the global stack.
3043 if (_cm->verbose_medium())
3044 gclog_or_tty->print_cr("[%d] task queue overflow, "
3045 "moving entries to the global stack",
3046 _task_id);
3047 move_entries_to_global_stack();
3049 // this should succeed since, even if we overflow the global
3050 // stack, we should have definitely removed some entries from the
3051 // local queue. So, there must be space on it.
3052 bool success = _task_queue->push(obj);
3053 tmp_guarantee_CM( success, "invariant" );
3054 }
3056 statsOnly( int tmp_size = _task_queue->size();
3057 if (tmp_size > _local_max_size)
3058 _local_max_size = tmp_size;
3059 ++_local_pushes );
3060 }
3062 void CMTask::reached_limit() {
3063 tmp_guarantee_CM( _words_scanned >= _words_scanned_limit ||
3064 _refs_reached >= _refs_reached_limit ,
3065 "shouldn't have been called otherwise" );
3066 regular_clock_call();
3067 }
3069 void CMTask::regular_clock_call() {
3070 if (has_aborted())
3071 return;
3073 // First, we need to recalculate the words scanned and refs reached
3074 // limits for the next clock call.
3075 recalculate_limits();
3077 // During the regular clock call we do the following
3079 // (1) If an overflow has been flagged, then we abort.
3080 if (_cm->has_overflown()) {
3081 set_has_aborted();
3082 return;
3083 }
3085 // If we are not concurrent (i.e. we're doing remark) we don't need
3086 // to check anything else. The other steps are only needed during
3087 // the concurrent marking phase.
3088 if (!concurrent())
3089 return;
3091 // (2) If marking has been aborted for Full GC, then we also abort.
3092 if (_cm->has_aborted()) {
3093 set_has_aborted();
3094 statsOnly( ++_aborted_cm_aborted );
3095 return;
3096 }
3098 double curr_time_ms = os::elapsedVTime() * 1000.0;
3100 // (3) If marking stats are enabled, then we update the step history.
3101 #if _MARKING_STATS_
3102 if (_words_scanned >= _words_scanned_limit)
3103 ++_clock_due_to_scanning;
3104 if (_refs_reached >= _refs_reached_limit)
3105 ++_clock_due_to_marking;
3107 double last_interval_ms = curr_time_ms - _interval_start_time_ms;
3108 _interval_start_time_ms = curr_time_ms;
3109 _all_clock_intervals_ms.add(last_interval_ms);
3111 if (_cm->verbose_medium()) {
3112 gclog_or_tty->print_cr("[%d] regular clock, interval = %1.2lfms, "
3113 "scanned = %d%s, refs reached = %d%s",
3114 _task_id, last_interval_ms,
3115 _words_scanned,
3116 (_words_scanned >= _words_scanned_limit) ? " (*)" : "",
3117 _refs_reached,
3118 (_refs_reached >= _refs_reached_limit) ? " (*)" : "");
3119 }
3120 #endif // _MARKING_STATS_
3122 // (4) We check whether we should yield. If we have to, then we abort.
3123 if (_cm->should_yield()) {
3124 // We should yield. To do this we abort the task. The caller is
3125 // responsible for yielding.
3126 set_has_aborted();
3127 statsOnly( ++_aborted_yield );
3128 return;
3129 }
3131 // (5) We check whether we've reached our time quota. If we have,
3132 // then we abort.
3133 double elapsed_time_ms = curr_time_ms - _start_time_ms;
3134 if (elapsed_time_ms > _time_target_ms) {
3135 set_has_aborted();
3136 _has_aborted_timed_out = true;
3137 statsOnly( ++_aborted_timed_out );
3138 return;
3139 }
3141 // (6) Finally, we check whether there are enough completed STAB
3142 // buffers available for processing. If there are, we abort.
3143 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3144 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) {
3145 if (_cm->verbose_low())
3146 gclog_or_tty->print_cr("[%d] aborting to deal with pending SATB buffers",
3147 _task_id);
3148 // we do need to process SATB buffers, we'll abort and restart
3149 // the marking task to do so
3150 set_has_aborted();
3151 statsOnly( ++_aborted_satb );
3152 return;
3153 }
3154 }
3156 void CMTask::recalculate_limits() {
3157 _real_words_scanned_limit = _words_scanned + words_scanned_period;
3158 _words_scanned_limit = _real_words_scanned_limit;
3160 _real_refs_reached_limit = _refs_reached + refs_reached_period;
3161 _refs_reached_limit = _real_refs_reached_limit;
3162 }
3164 void CMTask::decrease_limits() {
3165 // This is called when we believe that we're going to do an infrequent
3166 // operation which will increase the per byte scanned cost (i.e. move
3167 // entries to/from the global stack). It basically tries to decrease the
3168 // scanning limit so that the clock is called earlier.
3170 if (_cm->verbose_medium())
3171 gclog_or_tty->print_cr("[%d] decreasing limits", _task_id);
3173 _words_scanned_limit = _real_words_scanned_limit -
3174 3 * words_scanned_period / 4;
3175 _refs_reached_limit = _real_refs_reached_limit -
3176 3 * refs_reached_period / 4;
3177 }
3179 void CMTask::move_entries_to_global_stack() {
3180 // local array where we'll store the entries that will be popped
3181 // from the local queue
3182 oop buffer[global_stack_transfer_size];
3184 int n = 0;
3185 oop obj;
3186 while (n < global_stack_transfer_size && _task_queue->pop_local(obj)) {
3187 buffer[n] = obj;
3188 ++n;
3189 }
3191 if (n > 0) {
3192 // we popped at least one entry from the local queue
3194 statsOnly( ++_global_transfers_to; _local_pops += n );
3196 if (!_cm->mark_stack_push(buffer, n)) {
3197 if (_cm->verbose_low())
3198 gclog_or_tty->print_cr("[%d] aborting due to global stack overflow", _task_id);
3199 set_has_aborted();
3200 } else {
3201 // the transfer was successful
3203 if (_cm->verbose_medium())
3204 gclog_or_tty->print_cr("[%d] pushed %d entries to the global stack",
3205 _task_id, n);
3206 statsOnly( int tmp_size = _cm->mark_stack_size();
3207 if (tmp_size > _global_max_size)
3208 _global_max_size = tmp_size;
3209 _global_pushes += n );
3210 }
3211 }
3213 // this operation was quite expensive, so decrease the limits
3214 decrease_limits();
3215 }
3217 void CMTask::get_entries_from_global_stack() {
3218 // local array where we'll store the entries that will be popped
3219 // from the global stack.
3220 oop buffer[global_stack_transfer_size];
3221 int n;
3222 _cm->mark_stack_pop(buffer, global_stack_transfer_size, &n);
3223 tmp_guarantee_CM( n <= global_stack_transfer_size,
3224 "we should not pop more than the given limit" );
3225 if (n > 0) {
3226 // yes, we did actually pop at least one entry
3228 statsOnly( ++_global_transfers_from; _global_pops += n );
3229 if (_cm->verbose_medium())
3230 gclog_or_tty->print_cr("[%d] popped %d entries from the global stack",
3231 _task_id, n);
3232 for (int i = 0; i < n; ++i) {
3233 bool success = _task_queue->push(buffer[i]);
3234 // We only call this when the local queue is empty or under a
3235 // given target limit. So, we do not expect this push to fail.
3236 tmp_guarantee_CM( success, "invariant" );
3237 }
3239 statsOnly( int tmp_size = _task_queue->size();
3240 if (tmp_size > _local_max_size)
3241 _local_max_size = tmp_size;
3242 _local_pushes += n );
3243 }
3245 // this operation was quite expensive, so decrease the limits
3246 decrease_limits();
3247 }
3249 void CMTask::drain_local_queue(bool partially) {
3250 if (has_aborted())
3251 return;
3253 // Decide what the target size is, depending whether we're going to
3254 // drain it partially (so that other tasks can steal if they run out
3255 // of things to do) or totally (at the very end).
3256 size_t target_size;
3257 if (partially)
3258 target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize);
3259 else
3260 target_size = 0;
3262 if (_task_queue->size() > target_size) {
3263 if (_cm->verbose_high())
3264 gclog_or_tty->print_cr("[%d] draining local queue, target size = %d",
3265 _task_id, target_size);
3267 oop obj;
3268 bool ret = _task_queue->pop_local(obj);
3269 while (ret) {
3270 statsOnly( ++_local_pops );
3272 if (_cm->verbose_high())
3273 gclog_or_tty->print_cr("[%d] popped "PTR_FORMAT, _task_id,
3274 (void*) obj);
3276 tmp_guarantee_CM( _g1h->is_in_g1_reserved((HeapWord*) obj),
3277 "invariant" );
3279 scan_object(obj);
3281 if (_task_queue->size() <= target_size || has_aborted())
3282 ret = false;
3283 else
3284 ret = _task_queue->pop_local(obj);
3285 }
3287 if (_cm->verbose_high())
3288 gclog_or_tty->print_cr("[%d] drained local queue, size = %d",
3289 _task_id, _task_queue->size());
3290 }
3291 }
3293 void CMTask::drain_global_stack(bool partially) {
3294 if (has_aborted())
3295 return;
3297 // We have a policy to drain the local queue before we attempt to
3298 // drain the global stack.
3299 tmp_guarantee_CM( partially || _task_queue->size() == 0, "invariant" );
3301 // Decide what the target size is, depending whether we're going to
3302 // drain it partially (so that other tasks can steal if they run out
3303 // of things to do) or totally (at the very end). Notice that,
3304 // because we move entries from the global stack in chunks or
3305 // because another task might be doing the same, we might in fact
3306 // drop below the target. But, this is not a problem.
3307 size_t target_size;
3308 if (partially)
3309 target_size = _cm->partial_mark_stack_size_target();
3310 else
3311 target_size = 0;
3313 if (_cm->mark_stack_size() > target_size) {
3314 if (_cm->verbose_low())
3315 gclog_or_tty->print_cr("[%d] draining global_stack, target size %d",
3316 _task_id, target_size);
3318 while (!has_aborted() && _cm->mark_stack_size() > target_size) {
3319 get_entries_from_global_stack();
3320 drain_local_queue(partially);
3321 }
3323 if (_cm->verbose_low())
3324 gclog_or_tty->print_cr("[%d] drained global stack, size = %d",
3325 _task_id, _cm->mark_stack_size());
3326 }
3327 }
3329 // SATB Queue has several assumptions on whether to call the par or
3330 // non-par versions of the methods. this is why some of the code is
3331 // replicated. We should really get rid of the single-threaded version
3332 // of the code to simplify things.
3333 void CMTask::drain_satb_buffers() {
3334 if (has_aborted())
3335 return;
3337 // We set this so that the regular clock knows that we're in the
3338 // middle of draining buffers and doesn't set the abort flag when it
3339 // notices that SATB buffers are available for draining. It'd be
3340 // very counter productive if it did that. :-)
3341 _draining_satb_buffers = true;
3343 CMObjectClosure oc(this);
3344 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3345 if (ParallelGCThreads > 0)
3346 satb_mq_set.set_par_closure(_task_id, &oc);
3347 else
3348 satb_mq_set.set_closure(&oc);
3350 // This keeps claiming and applying the closure to completed buffers
3351 // until we run out of buffers or we need to abort.
3352 if (ParallelGCThreads > 0) {
3353 while (!has_aborted() &&
3354 satb_mq_set.par_apply_closure_to_completed_buffer(_task_id)) {
3355 if (_cm->verbose_medium())
3356 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id);
3357 statsOnly( ++_satb_buffers_processed );
3358 regular_clock_call();
3359 }
3360 } else {
3361 while (!has_aborted() &&
3362 satb_mq_set.apply_closure_to_completed_buffer()) {
3363 if (_cm->verbose_medium())
3364 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id);
3365 statsOnly( ++_satb_buffers_processed );
3366 regular_clock_call();
3367 }
3368 }
3370 if (!concurrent() && !has_aborted()) {
3371 // We should only do this during remark.
3372 if (ParallelGCThreads > 0)
3373 satb_mq_set.par_iterate_closure_all_threads(_task_id);
3374 else
3375 satb_mq_set.iterate_closure_all_threads();
3376 }
3378 _draining_satb_buffers = false;
3380 tmp_guarantee_CM( has_aborted() ||
3381 concurrent() ||
3382 satb_mq_set.completed_buffers_num() == 0, "invariant" );
3384 if (ParallelGCThreads > 0)
3385 satb_mq_set.set_par_closure(_task_id, NULL);
3386 else
3387 satb_mq_set.set_closure(NULL);
3389 // again, this was a potentially expensive operation, decrease the
3390 // limits to get the regular clock call early
3391 decrease_limits();
3392 }
3394 void CMTask::drain_region_stack(BitMapClosure* bc) {
3395 if (has_aborted())
3396 return;
3398 tmp_guarantee_CM( _region_finger == NULL,
3399 "it should be NULL when we're not scanning a region" );
3401 if (!_cm->region_stack_empty()) {
3402 if (_cm->verbose_low())
3403 gclog_or_tty->print_cr("[%d] draining region stack, size = %d",
3404 _task_id, _cm->region_stack_size());
3406 MemRegion mr = _cm->region_stack_pop();
3407 // it returns MemRegion() if the pop fails
3408 statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
3410 while (mr.start() != NULL) {
3411 if (_cm->verbose_medium())
3412 gclog_or_tty->print_cr("[%d] we are scanning region "
3413 "["PTR_FORMAT", "PTR_FORMAT")",
3414 _task_id, mr.start(), mr.end());
3415 tmp_guarantee_CM( mr.end() <= _cm->finger(),
3416 "otherwise the region shouldn't be on the stack" );
3417 assert(!mr.is_empty(), "Only non-empty regions live on the region stack");
3418 if (_nextMarkBitMap->iterate(bc, mr)) {
3419 tmp_guarantee_CM( !has_aborted(),
3420 "cannot abort the task without aborting the bitmap iteration" );
3422 // We finished iterating over the region without aborting.
3423 regular_clock_call();
3424 if (has_aborted())
3425 mr = MemRegion();
3426 else {
3427 mr = _cm->region_stack_pop();
3428 // it returns MemRegion() if the pop fails
3429 statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
3430 }
3431 } else {
3432 guarantee( has_aborted(), "currently the only way to do so" );
3434 // The only way to abort the bitmap iteration is to return
3435 // false from the do_bit() method. However, inside the
3436 // do_bit() method we move the _region_finger to point to the
3437 // object currently being looked at. So, if we bail out, we
3438 // have definitely set _region_finger to something non-null.
3439 guarantee( _region_finger != NULL, "invariant" );
3441 // The iteration was actually aborted. So now _region_finger
3442 // points to the address of the object we last scanned. If we
3443 // leave it there, when we restart this task, we will rescan
3444 // the object. It is easy to avoid this. We move the finger by
3445 // enough to point to the next possible object header (the
3446 // bitmap knows by how much we need to move it as it knows its
3447 // granularity).
3448 MemRegion newRegion =
3449 MemRegion(_nextMarkBitMap->nextWord(_region_finger), mr.end());
3451 if (!newRegion.is_empty()) {
3452 if (_cm->verbose_low()) {
3453 gclog_or_tty->print_cr("[%d] pushing unscanned region"
3454 "[" PTR_FORMAT "," PTR_FORMAT ") on region stack",
3455 _task_id,
3456 newRegion.start(), newRegion.end());
3457 }
3458 // Now push the part of the region we didn't scan on the
3459 // region stack to make sure a task scans it later.
3460 _cm->region_stack_push(newRegion);
3461 }
3462 // break from while
3463 mr = MemRegion();
3464 }
3465 _region_finger = NULL;
3466 }
3468 // We only push regions on the region stack during evacuation
3469 // pauses. So if we come out the above iteration because we region
3470 // stack is empty, it will remain empty until the next yield
3471 // point. So, the guarantee below is safe.
3472 guarantee( has_aborted() || _cm->region_stack_empty(),
3473 "only way to exit the loop" );
3475 if (_cm->verbose_low())
3476 gclog_or_tty->print_cr("[%d] drained region stack, size = %d",
3477 _task_id, _cm->region_stack_size());
3478 }
3479 }
3481 void CMTask::print_stats() {
3482 gclog_or_tty->print_cr("Marking Stats, task = %d, calls = %d",
3483 _task_id, _calls);
3484 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3485 _elapsed_time_ms, _termination_time_ms);
3486 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3487 _step_times_ms.num(), _step_times_ms.avg(),
3488 _step_times_ms.sd());
3489 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
3490 _step_times_ms.maximum(), _step_times_ms.sum());
3492 #if _MARKING_STATS_
3493 gclog_or_tty->print_cr(" Clock Intervals (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3494 _all_clock_intervals_ms.num(), _all_clock_intervals_ms.avg(),
3495 _all_clock_intervals_ms.sd());
3496 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
3497 _all_clock_intervals_ms.maximum(),
3498 _all_clock_intervals_ms.sum());
3499 gclog_or_tty->print_cr(" Clock Causes (cum): scanning = %d, marking = %d",
3500 _clock_due_to_scanning, _clock_due_to_marking);
3501 gclog_or_tty->print_cr(" Objects: scanned = %d, found on the bitmap = %d",
3502 _objs_scanned, _objs_found_on_bitmap);
3503 gclog_or_tty->print_cr(" Local Queue: pushes = %d, pops = %d, max size = %d",
3504 _local_pushes, _local_pops, _local_max_size);
3505 gclog_or_tty->print_cr(" Global Stack: pushes = %d, pops = %d, max size = %d",
3506 _global_pushes, _global_pops, _global_max_size);
3507 gclog_or_tty->print_cr(" transfers to = %d, transfers from = %d",
3508 _global_transfers_to,_global_transfers_from);
3509 gclog_or_tty->print_cr(" Regions: claimed = %d, Region Stack: pops = %d",
3510 _regions_claimed, _region_stack_pops);
3511 gclog_or_tty->print_cr(" SATB buffers: processed = %d", _satb_buffers_processed);
3512 gclog_or_tty->print_cr(" Steals: attempts = %d, successes = %d",
3513 _steal_attempts, _steals);
3514 gclog_or_tty->print_cr(" Aborted: %d, due to", _aborted);
3515 gclog_or_tty->print_cr(" overflow: %d, global abort: %d, yield: %d",
3516 _aborted_overflow, _aborted_cm_aborted, _aborted_yield);
3517 gclog_or_tty->print_cr(" time out: %d, SATB: %d, termination: %d",
3518 _aborted_timed_out, _aborted_satb, _aborted_termination);
3519 #endif // _MARKING_STATS_
3520 }
3522 /*****************************************************************************
3524 The do_marking_step(time_target_ms) method is the building block
3525 of the parallel marking framework. It can be called in parallel
3526 with other invocations of do_marking_step() on different tasks
3527 (but only one per task, obviously) and concurrently with the
3528 mutator threads, or during remark, hence it eliminates the need
3529 for two versions of the code. When called during remark, it will
3530 pick up from where the task left off during the concurrent marking
3531 phase. Interestingly, tasks are also claimable during evacuation
3532 pauses too, since do_marking_step() ensures that it aborts before
3533 it needs to yield.
3535 The data structures that is uses to do marking work are the
3536 following:
3538 (1) Marking Bitmap. If there are gray objects that appear only
3539 on the bitmap (this happens either when dealing with an overflow
3540 or when the initial marking phase has simply marked the roots
3541 and didn't push them on the stack), then tasks claim heap
3542 regions whose bitmap they then scan to find gray objects. A
3543 global finger indicates where the end of the last claimed region
3544 is. A local finger indicates how far into the region a task has
3545 scanned. The two fingers are used to determine how to gray an
3546 object (i.e. whether simply marking it is OK, as it will be
3547 visited by a task in the future, or whether it needs to be also
3548 pushed on a stack).
3550 (2) Local Queue. The local queue of the task which is accessed
3551 reasonably efficiently by the task. Other tasks can steal from
3552 it when they run out of work. Throughout the marking phase, a
3553 task attempts to keep its local queue short but not totally
3554 empty, so that entries are available for stealing by other
3555 tasks. Only when there is no more work, a task will totally
3556 drain its local queue.
3558 (3) Global Mark Stack. This handles local queue overflow. During
3559 marking only sets of entries are moved between it and the local
3560 queues, as access to it requires a mutex and more fine-grain
3561 interaction with it which might cause contention. If it
3562 overflows, then the marking phase should restart and iterate
3563 over the bitmap to identify gray objects. Throughout the marking
3564 phase, tasks attempt to keep the global mark stack at a small
3565 length but not totally empty, so that entries are available for
3566 popping by other tasks. Only when there is no more work, tasks
3567 will totally drain the global mark stack.
3569 (4) Global Region Stack. Entries on it correspond to areas of
3570 the bitmap that need to be scanned since they contain gray
3571 objects. Pushes on the region stack only happen during
3572 evacuation pauses and typically correspond to areas covered by
3573 GC LABS. If it overflows, then the marking phase should restart
3574 and iterate over the bitmap to identify gray objects. Tasks will
3575 try to totally drain the region stack as soon as possible.
3577 (5) SATB Buffer Queue. This is where completed SATB buffers are
3578 made available. Buffers are regularly removed from this queue
3579 and scanned for roots, so that the queue doesn't get too
3580 long. During remark, all completed buffers are processed, as
3581 well as the filled in parts of any uncompleted buffers.
3583 The do_marking_step() method tries to abort when the time target
3584 has been reached. There are a few other cases when the
3585 do_marking_step() method also aborts:
3587 (1) When the marking phase has been aborted (after a Full GC).
3589 (2) When a global overflow (either on the global stack or the
3590 region stack) has been triggered. Before the task aborts, it
3591 will actually sync up with the other tasks to ensure that all
3592 the marking data structures (local queues, stacks, fingers etc.)
3593 are re-initialised so that when do_marking_step() completes,
3594 the marking phase can immediately restart.
3596 (3) When enough completed SATB buffers are available. The
3597 do_marking_step() method only tries to drain SATB buffers right
3598 at the beginning. So, if enough buffers are available, the
3599 marking step aborts and the SATB buffers are processed at
3600 the beginning of the next invocation.
3602 (4) To yield. when we have to yield then we abort and yield
3603 right at the end of do_marking_step(). This saves us from a lot
3604 of hassle as, by yielding we might allow a Full GC. If this
3605 happens then objects will be compacted underneath our feet, the
3606 heap might shrink, etc. We save checking for this by just
3607 aborting and doing the yield right at the end.
3609 From the above it follows that the do_marking_step() method should
3610 be called in a loop (or, otherwise, regularly) until it completes.
3612 If a marking step completes without its has_aborted() flag being
3613 true, it means it has completed the current marking phase (and
3614 also all other marking tasks have done so and have all synced up).
3616 A method called regular_clock_call() is invoked "regularly" (in
3617 sub ms intervals) throughout marking. It is this clock method that
3618 checks all the abort conditions which were mentioned above and
3619 decides when the task should abort. A work-based scheme is used to
3620 trigger this clock method: when the number of object words the
3621 marking phase has scanned or the number of references the marking
3622 phase has visited reach a given limit. Additional invocations to
3623 the method clock have been planted in a few other strategic places
3624 too. The initial reason for the clock method was to avoid calling
3625 vtime too regularly, as it is quite expensive. So, once it was in
3626 place, it was natural to piggy-back all the other conditions on it
3627 too and not constantly check them throughout the code.
3629 *****************************************************************************/
3631 void CMTask::do_marking_step(double time_target_ms) {
3632 guarantee( time_target_ms >= 1.0, "minimum granularity is 1ms" );
3633 guarantee( concurrent() == _cm->concurrent(), "they should be the same" );
3635 guarantee( concurrent() || _cm->region_stack_empty(),
3636 "the region stack should have been cleared before remark" );
3637 guarantee( _region_finger == NULL,
3638 "this should be non-null only when a region is being scanned" );
3640 G1CollectorPolicy* g1_policy = _g1h->g1_policy();
3641 guarantee( _task_queues != NULL, "invariant" );
3642 guarantee( _task_queue != NULL, "invariant" );
3643 guarantee( _task_queues->queue(_task_id) == _task_queue, "invariant" );
3645 guarantee( !_claimed,
3646 "only one thread should claim this task at any one time" );
3648 // OK, this doesn't safeguard again all possible scenarios, as it is
3649 // possible for two threads to set the _claimed flag at the same
3650 // time. But it is only for debugging purposes anyway and it will
3651 // catch most problems.
3652 _claimed = true;
3654 _start_time_ms = os::elapsedVTime() * 1000.0;
3655 statsOnly( _interval_start_time_ms = _start_time_ms );
3657 double diff_prediction_ms =
3658 g1_policy->get_new_prediction(&_marking_step_diffs_ms);
3659 _time_target_ms = time_target_ms - diff_prediction_ms;
3661 // set up the variables that are used in the work-based scheme to
3662 // call the regular clock method
3663 _words_scanned = 0;
3664 _refs_reached = 0;
3665 recalculate_limits();
3667 // clear all flags
3668 clear_has_aborted();
3669 _has_aborted_timed_out = false;
3670 _draining_satb_buffers = false;
3672 ++_calls;
3674 if (_cm->verbose_low())
3675 gclog_or_tty->print_cr("[%d] >>>>>>>>>> START, call = %d, "
3676 "target = %1.2lfms >>>>>>>>>>",
3677 _task_id, _calls, _time_target_ms);
3679 // Set up the bitmap and oop closures. Anything that uses them is
3680 // eventually called from this method, so it is OK to allocate these
3681 // statically.
3682 CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap);
3683 CMOopClosure oop_closure(_g1h, _cm, this);
3684 set_oop_closure(&oop_closure);
3686 if (_cm->has_overflown()) {
3687 // This can happen if the region stack or the mark stack overflows
3688 // during a GC pause and this task, after a yield point,
3689 // restarts. We have to abort as we need to get into the overflow
3690 // protocol which happens right at the end of this task.
3691 set_has_aborted();
3692 }
3694 // First drain any available SATB buffers. After this, we will not
3695 // look at SATB buffers before the next invocation of this method.
3696 // If enough completed SATB buffers are queued up, the regular clock
3697 // will abort this task so that it restarts.
3698 drain_satb_buffers();
3699 // ...then partially drain the local queue and the global stack
3700 drain_local_queue(true);
3701 drain_global_stack(true);
3703 // Then totally drain the region stack. We will not look at
3704 // it again before the next invocation of this method. Entries on
3705 // the region stack are only added during evacuation pauses, for
3706 // which we have to yield. When we do, we abort the task anyway so
3707 // it will look at the region stack again when it restarts.
3708 bitmap_closure.set_scanning_heap_region(false);
3709 drain_region_stack(&bitmap_closure);
3710 // ...then partially drain the local queue and the global stack
3711 drain_local_queue(true);
3712 drain_global_stack(true);
3714 do {
3715 if (!has_aborted() && _curr_region != NULL) {
3716 // This means that we're already holding on to a region.
3717 tmp_guarantee_CM( _finger != NULL,
3718 "if region is not NULL, then the finger "
3719 "should not be NULL either" );
3721 // We might have restarted this task after an evacuation pause
3722 // which might have evacuated the region we're holding on to
3723 // underneath our feet. Let's read its limit again to make sure
3724 // that we do not iterate over a region of the heap that
3725 // contains garbage (update_region_limit() will also move
3726 // _finger to the start of the region if it is found empty).
3727 update_region_limit();
3728 // We will start from _finger not from the start of the region,
3729 // as we might be restarting this task after aborting half-way
3730 // through scanning this region. In this case, _finger points to
3731 // the address where we last found a marked object. If this is a
3732 // fresh region, _finger points to start().
3733 MemRegion mr = MemRegion(_finger, _region_limit);
3735 if (_cm->verbose_low())
3736 gclog_or_tty->print_cr("[%d] we're scanning part "
3737 "["PTR_FORMAT", "PTR_FORMAT") "
3738 "of region "PTR_FORMAT,
3739 _task_id, _finger, _region_limit, _curr_region);
3741 // Let's iterate over the bitmap of the part of the
3742 // region that is left.
3743 bitmap_closure.set_scanning_heap_region(true);
3744 if (mr.is_empty() ||
3745 _nextMarkBitMap->iterate(&bitmap_closure, mr)) {
3746 // We successfully completed iterating over the region. Now,
3747 // let's give up the region.
3748 giveup_current_region();
3749 regular_clock_call();
3750 } else {
3751 guarantee( has_aborted(), "currently the only way to do so" );
3752 // The only way to abort the bitmap iteration is to return
3753 // false from the do_bit() method. However, inside the
3754 // do_bit() method we move the _finger to point to the
3755 // object currently being looked at. So, if we bail out, we
3756 // have definitely set _finger to something non-null.
3757 guarantee( _finger != NULL, "invariant" );
3759 // Region iteration was actually aborted. So now _finger
3760 // points to the address of the object we last scanned. If we
3761 // leave it there, when we restart this task, we will rescan
3762 // the object. It is easy to avoid this. We move the finger by
3763 // enough to point to the next possible object header (the
3764 // bitmap knows by how much we need to move it as it knows its
3765 // granularity).
3766 move_finger_to(_nextMarkBitMap->nextWord(_finger));
3767 }
3768 }
3769 // At this point we have either completed iterating over the
3770 // region we were holding on to, or we have aborted.
3772 // We then partially drain the local queue and the global stack.
3773 // (Do we really need this?)
3774 drain_local_queue(true);
3775 drain_global_stack(true);
3777 // Read the note on the claim_region() method on why it might
3778 // return NULL with potentially more regions available for
3779 // claiming and why we have to check out_of_regions() to determine
3780 // whether we're done or not.
3781 while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) {
3782 // We are going to try to claim a new region. We should have
3783 // given up on the previous one.
3784 tmp_guarantee_CM( _curr_region == NULL &&
3785 _finger == NULL &&
3786 _region_limit == NULL, "invariant" );
3787 if (_cm->verbose_low())
3788 gclog_or_tty->print_cr("[%d] trying to claim a new region", _task_id);
3789 HeapRegion* claimed_region = _cm->claim_region(_task_id);
3790 if (claimed_region != NULL) {
3791 // Yes, we managed to claim one
3792 statsOnly( ++_regions_claimed );
3794 if (_cm->verbose_low())
3795 gclog_or_tty->print_cr("[%d] we successfully claimed "
3796 "region "PTR_FORMAT,
3797 _task_id, claimed_region);
3799 setup_for_region(claimed_region);
3800 tmp_guarantee_CM( _curr_region == claimed_region, "invariant" );
3801 }
3802 // It is important to call the regular clock here. It might take
3803 // a while to claim a region if, for example, we hit a large
3804 // block of empty regions. So we need to call the regular clock
3805 // method once round the loop to make sure it's called
3806 // frequently enough.
3807 regular_clock_call();
3808 }
3810 if (!has_aborted() && _curr_region == NULL) {
3811 tmp_guarantee_CM( _cm->out_of_regions(),
3812 "at this point we should be out of regions" );
3813 }
3814 } while ( _curr_region != NULL && !has_aborted());
3816 if (!has_aborted()) {
3817 // We cannot check whether the global stack is empty, since other
3818 // tasks might be pushing objects to it concurrently. We also cannot
3819 // check if the region stack is empty because if a thread is aborting
3820 // it can push a partially done region back.
3821 tmp_guarantee_CM( _cm->out_of_regions(),
3822 "at this point we should be out of regions" );
3824 if (_cm->verbose_low())
3825 gclog_or_tty->print_cr("[%d] all regions claimed", _task_id);
3827 // Try to reduce the number of available SATB buffers so that
3828 // remark has less work to do.
3829 drain_satb_buffers();
3830 }
3832 // Since we've done everything else, we can now totally drain the
3833 // local queue and global stack.
3834 drain_local_queue(false);
3835 drain_global_stack(false);
3837 // Attempt at work stealing from other task's queues.
3838 if (!has_aborted()) {
3839 // We have not aborted. This means that we have finished all that
3840 // we could. Let's try to do some stealing...
3842 // We cannot check whether the global stack is empty, since other
3843 // tasks might be pushing objects to it concurrently. We also cannot
3844 // check if the region stack is empty because if a thread is aborting
3845 // it can push a partially done region back.
3846 guarantee( _cm->out_of_regions() &&
3847 _task_queue->size() == 0, "only way to reach here" );
3849 if (_cm->verbose_low())
3850 gclog_or_tty->print_cr("[%d] starting to steal", _task_id);
3852 while (!has_aborted()) {
3853 oop obj;
3854 statsOnly( ++_steal_attempts );
3856 if (_cm->try_stealing(_task_id, &_hash_seed, obj)) {
3857 if (_cm->verbose_medium())
3858 gclog_or_tty->print_cr("[%d] stolen "PTR_FORMAT" successfully",
3859 _task_id, (void*) obj);
3861 statsOnly( ++_steals );
3863 tmp_guarantee_CM( _nextMarkBitMap->isMarked((HeapWord*) obj),
3864 "any stolen object should be marked" );
3865 scan_object(obj);
3867 // And since we're towards the end, let's totally drain the
3868 // local queue and global stack.
3869 drain_local_queue(false);
3870 drain_global_stack(false);
3871 } else {
3872 break;
3873 }
3874 }
3875 }
3877 // We still haven't aborted. Now, let's try to get into the
3878 // termination protocol.
3879 if (!has_aborted()) {
3880 // We cannot check whether the global stack is empty, since other
3881 // tasks might be concurrently pushing objects on it. We also cannot
3882 // check if the region stack is empty because if a thread is aborting
3883 // it can push a partially done region back.
3884 guarantee( _cm->out_of_regions() &&
3885 _task_queue->size() == 0, "only way to reach here" );
3887 if (_cm->verbose_low())
3888 gclog_or_tty->print_cr("[%d] starting termination protocol", _task_id);
3890 _termination_start_time_ms = os::elapsedVTime() * 1000.0;
3891 // The CMTask class also extends the TerminatorTerminator class,
3892 // hence its should_exit_termination() method will also decide
3893 // whether to exit the termination protocol or not.
3894 bool finished = _cm->terminator()->offer_termination(this);
3895 double termination_end_time_ms = os::elapsedVTime() * 1000.0;
3896 _termination_time_ms +=
3897 termination_end_time_ms - _termination_start_time_ms;
3899 if (finished) {
3900 // We're all done.
3902 if (_task_id == 0) {
3903 // let's allow task 0 to do this
3904 if (concurrent()) {
3905 guarantee( _cm->concurrent_marking_in_progress(), "invariant" );
3906 // we need to set this to false before the next
3907 // safepoint. This way we ensure that the marking phase
3908 // doesn't observe any more heap expansions.
3909 _cm->clear_concurrent_marking_in_progress();
3910 }
3911 }
3913 // We can now guarantee that the global stack is empty, since
3914 // all other tasks have finished.
3915 guarantee( _cm->out_of_regions() &&
3916 _cm->region_stack_empty() &&
3917 _cm->mark_stack_empty() &&
3918 _task_queue->size() == 0 &&
3919 !_cm->has_overflown() &&
3920 !_cm->mark_stack_overflow() &&
3921 !_cm->region_stack_overflow(),
3922 "only way to reach here" );
3924 if (_cm->verbose_low())
3925 gclog_or_tty->print_cr("[%d] all tasks terminated", _task_id);
3926 } else {
3927 // Apparently there's more work to do. Let's abort this task. It
3928 // will restart it and we can hopefully find more things to do.
3930 if (_cm->verbose_low())
3931 gclog_or_tty->print_cr("[%d] apparently there is more work to do", _task_id);
3933 set_has_aborted();
3934 statsOnly( ++_aborted_termination );
3935 }
3936 }
3938 // Mainly for debugging purposes to make sure that a pointer to the
3939 // closure which was statically allocated in this frame doesn't
3940 // escape it by accident.
3941 set_oop_closure(NULL);
3942 double end_time_ms = os::elapsedVTime() * 1000.0;
3943 double elapsed_time_ms = end_time_ms - _start_time_ms;
3944 // Update the step history.
3945 _step_times_ms.add(elapsed_time_ms);
3947 if (has_aborted()) {
3948 // The task was aborted for some reason.
3950 statsOnly( ++_aborted );
3952 if (_has_aborted_timed_out) {
3953 double diff_ms = elapsed_time_ms - _time_target_ms;
3954 // Keep statistics of how well we did with respect to hitting
3955 // our target only if we actually timed out (if we aborted for
3956 // other reasons, then the results might get skewed).
3957 _marking_step_diffs_ms.add(diff_ms);
3958 }
3960 if (_cm->has_overflown()) {
3961 // This is the interesting one. We aborted because a global
3962 // overflow was raised. This means we have to restart the
3963 // marking phase and start iterating over regions. However, in
3964 // order to do this we have to make sure that all tasks stop
3965 // what they are doing and re-initialise in a safe manner. We
3966 // will achieve this with the use of two barrier sync points.
3968 if (_cm->verbose_low())
3969 gclog_or_tty->print_cr("[%d] detected overflow", _task_id);
3971 _cm->enter_first_sync_barrier(_task_id);
3972 // When we exit this sync barrier we know that all tasks have
3973 // stopped doing marking work. So, it's now safe to
3974 // re-initialise our data structures. At the end of this method,
3975 // task 0 will clear the global data structures.
3977 statsOnly( ++_aborted_overflow );
3979 // We clear the local state of this task...
3980 clear_region_fields();
3982 // ...and enter the second barrier.
3983 _cm->enter_second_sync_barrier(_task_id);
3984 // At this point everything has bee re-initialised and we're
3985 // ready to restart.
3986 }
3988 if (_cm->verbose_low()) {
3989 gclog_or_tty->print_cr("[%d] <<<<<<<<<< ABORTING, target = %1.2lfms, "
3990 "elapsed = %1.2lfms <<<<<<<<<<",
3991 _task_id, _time_target_ms, elapsed_time_ms);
3992 if (_cm->has_aborted())
3993 gclog_or_tty->print_cr("[%d] ========== MARKING ABORTED ==========",
3994 _task_id);
3995 }
3996 } else {
3997 if (_cm->verbose_low())
3998 gclog_or_tty->print_cr("[%d] <<<<<<<<<< FINISHED, target = %1.2lfms, "
3999 "elapsed = %1.2lfms <<<<<<<<<<",
4000 _task_id, _time_target_ms, elapsed_time_ms);
4001 }
4003 _claimed = false;
4004 }
4006 CMTask::CMTask(int task_id,
4007 ConcurrentMark* cm,
4008 CMTaskQueue* task_queue,
4009 CMTaskQueueSet* task_queues)
4010 : _g1h(G1CollectedHeap::heap()),
4011 _co_tracker(G1CMGroup),
4012 _task_id(task_id), _cm(cm),
4013 _claimed(false),
4014 _nextMarkBitMap(NULL), _hash_seed(17),
4015 _task_queue(task_queue),
4016 _task_queues(task_queues),
4017 _oop_closure(NULL) {
4018 guarantee( task_queue != NULL, "invariant" );
4019 guarantee( task_queues != NULL, "invariant" );
4021 statsOnly( _clock_due_to_scanning = 0;
4022 _clock_due_to_marking = 0 );
4024 _marking_step_diffs_ms.add(0.5);
4025 }