Tue, 25 Jan 2011 10:56:22 -0800
6608385: G1: need to support parallel reference processing
Summary: Implement support for ParallelRefProcEnabled in the reference processing that takes place at the end of G1 concurrent marking.
Reviewed-by: tonyp, ysr
1 /*
2 * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
25 #include "precompiled.hpp"
26 #include "classfile/symbolTable.hpp"
27 #include "gc_implementation/g1/concurrentMark.hpp"
28 #include "gc_implementation/g1/concurrentMarkThread.inline.hpp"
29 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
30 #include "gc_implementation/g1/g1CollectorPolicy.hpp"
31 #include "gc_implementation/g1/g1RemSet.hpp"
32 #include "gc_implementation/g1/heapRegionRemSet.hpp"
33 #include "gc_implementation/g1/heapRegionSeq.inline.hpp"
34 #include "gc_implementation/shared/vmGCOperations.hpp"
35 #include "memory/genOopClosures.inline.hpp"
36 #include "memory/referencePolicy.hpp"
37 #include "memory/resourceArea.hpp"
38 #include "oops/oop.inline.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/java.hpp"
42 //
43 // CMS Bit Map Wrapper
45 CMBitMapRO::CMBitMapRO(ReservedSpace rs, int shifter):
46 _bm((uintptr_t*)NULL,0),
47 _shifter(shifter) {
48 _bmStartWord = (HeapWord*)(rs.base());
49 _bmWordSize = rs.size()/HeapWordSize; // rs.size() is in bytes
50 ReservedSpace brs(ReservedSpace::allocation_align_size_up(
51 (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1));
53 guarantee(brs.is_reserved(), "couldn't allocate CMS bit map");
54 // For now we'll just commit all of the bit map up fromt.
55 // Later on we'll try to be more parsimonious with swap.
56 guarantee(_virtual_space.initialize(brs, brs.size()),
57 "couldn't reseve backing store for CMS bit map");
58 assert(_virtual_space.committed_size() == brs.size(),
59 "didn't reserve backing store for all of CMS bit map?");
60 _bm.set_map((uintptr_t*)_virtual_space.low());
61 assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
62 _bmWordSize, "inconsistency in bit map sizing");
63 _bm.set_size(_bmWordSize >> _shifter);
64 }
66 HeapWord* CMBitMapRO::getNextMarkedWordAddress(HeapWord* addr,
67 HeapWord* limit) const {
68 // First we must round addr *up* to a possible object boundary.
69 addr = (HeapWord*)align_size_up((intptr_t)addr,
70 HeapWordSize << _shifter);
71 size_t addrOffset = heapWordToOffset(addr);
72 if (limit == NULL) limit = _bmStartWord + _bmWordSize;
73 size_t limitOffset = heapWordToOffset(limit);
74 size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset);
75 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
76 assert(nextAddr >= addr, "get_next_one postcondition");
77 assert(nextAddr == limit || isMarked(nextAddr),
78 "get_next_one postcondition");
79 return nextAddr;
80 }
82 HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(HeapWord* addr,
83 HeapWord* limit) const {
84 size_t addrOffset = heapWordToOffset(addr);
85 if (limit == NULL) limit = _bmStartWord + _bmWordSize;
86 size_t limitOffset = heapWordToOffset(limit);
87 size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset);
88 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
89 assert(nextAddr >= addr, "get_next_one postcondition");
90 assert(nextAddr == limit || !isMarked(nextAddr),
91 "get_next_one postcondition");
92 return nextAddr;
93 }
95 int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const {
96 assert((diff & ((1 << _shifter) - 1)) == 0, "argument check");
97 return (int) (diff >> _shifter);
98 }
100 bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) {
101 HeapWord* left = MAX2(_bmStartWord, mr.start());
102 HeapWord* right = MIN2(_bmStartWord + _bmWordSize, mr.end());
103 if (right > left) {
104 // Right-open interval [leftOffset, rightOffset).
105 return _bm.iterate(cl, heapWordToOffset(left), heapWordToOffset(right));
106 } else {
107 return true;
108 }
109 }
111 void CMBitMapRO::mostly_disjoint_range_union(BitMap* from_bitmap,
112 size_t from_start_index,
113 HeapWord* to_start_word,
114 size_t word_num) {
115 _bm.mostly_disjoint_range_union(from_bitmap,
116 from_start_index,
117 heapWordToOffset(to_start_word),
118 word_num);
119 }
121 #ifndef PRODUCT
122 bool CMBitMapRO::covers(ReservedSpace rs) const {
123 // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
124 assert(((size_t)_bm.size() * (size_t)(1 << _shifter)) == _bmWordSize,
125 "size inconsistency");
126 return _bmStartWord == (HeapWord*)(rs.base()) &&
127 _bmWordSize == rs.size()>>LogHeapWordSize;
128 }
129 #endif
131 void CMBitMap::clearAll() {
132 _bm.clear();
133 return;
134 }
136 void CMBitMap::markRange(MemRegion mr) {
137 mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
138 assert(!mr.is_empty(), "unexpected empty region");
139 assert((offsetToHeapWord(heapWordToOffset(mr.end())) ==
140 ((HeapWord *) mr.end())),
141 "markRange memory region end is not card aligned");
142 // convert address range into offset range
143 _bm.at_put_range(heapWordToOffset(mr.start()),
144 heapWordToOffset(mr.end()), true);
145 }
147 void CMBitMap::clearRange(MemRegion mr) {
148 mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
149 assert(!mr.is_empty(), "unexpected empty region");
150 // convert address range into offset range
151 _bm.at_put_range(heapWordToOffset(mr.start()),
152 heapWordToOffset(mr.end()), false);
153 }
155 MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr,
156 HeapWord* end_addr) {
157 HeapWord* start = getNextMarkedWordAddress(addr);
158 start = MIN2(start, end_addr);
159 HeapWord* end = getNextUnmarkedWordAddress(start);
160 end = MIN2(end, end_addr);
161 assert(start <= end, "Consistency check");
162 MemRegion mr(start, end);
163 if (!mr.is_empty()) {
164 clearRange(mr);
165 }
166 return mr;
167 }
169 CMMarkStack::CMMarkStack(ConcurrentMark* cm) :
170 _base(NULL), _cm(cm)
171 #ifdef ASSERT
172 , _drain_in_progress(false)
173 , _drain_in_progress_yields(false)
174 #endif
175 {}
177 void CMMarkStack::allocate(size_t size) {
178 _base = NEW_C_HEAP_ARRAY(oop, size);
179 if (_base == NULL)
180 vm_exit_during_initialization("Failed to allocate "
181 "CM region mark stack");
182 _index = 0;
183 // QQQQ cast ...
184 _capacity = (jint) size;
185 _oops_do_bound = -1;
186 NOT_PRODUCT(_max_depth = 0);
187 }
189 CMMarkStack::~CMMarkStack() {
190 if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base);
191 }
193 void CMMarkStack::par_push(oop ptr) {
194 while (true) {
195 if (isFull()) {
196 _overflow = true;
197 return;
198 }
199 // Otherwise...
200 jint index = _index;
201 jint next_index = index+1;
202 jint res = Atomic::cmpxchg(next_index, &_index, index);
203 if (res == index) {
204 _base[index] = ptr;
205 // Note that we don't maintain this atomically. We could, but it
206 // doesn't seem necessary.
207 NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index));
208 return;
209 }
210 // Otherwise, we need to try again.
211 }
212 }
214 void CMMarkStack::par_adjoin_arr(oop* ptr_arr, int n) {
215 while (true) {
216 if (isFull()) {
217 _overflow = true;
218 return;
219 }
220 // Otherwise...
221 jint index = _index;
222 jint next_index = index + n;
223 if (next_index > _capacity) {
224 _overflow = true;
225 return;
226 }
227 jint res = Atomic::cmpxchg(next_index, &_index, index);
228 if (res == index) {
229 for (int i = 0; i < n; i++) {
230 int ind = index + i;
231 assert(ind < _capacity, "By overflow test above.");
232 _base[ind] = ptr_arr[i];
233 }
234 NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index));
235 return;
236 }
237 // Otherwise, we need to try again.
238 }
239 }
242 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
243 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
244 jint start = _index;
245 jint next_index = start + n;
246 if (next_index > _capacity) {
247 _overflow = true;
248 return;
249 }
250 // Otherwise.
251 _index = next_index;
252 for (int i = 0; i < n; i++) {
253 int ind = start + i;
254 assert(ind < _capacity, "By overflow test above.");
255 _base[ind] = ptr_arr[i];
256 }
257 }
260 bool CMMarkStack::par_pop_arr(oop* ptr_arr, int max, int* n) {
261 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
262 jint index = _index;
263 if (index == 0) {
264 *n = 0;
265 return false;
266 } else {
267 int k = MIN2(max, index);
268 jint new_ind = index - k;
269 for (int j = 0; j < k; j++) {
270 ptr_arr[j] = _base[new_ind + j];
271 }
272 _index = new_ind;
273 *n = k;
274 return true;
275 }
276 }
279 CMRegionStack::CMRegionStack() : _base(NULL) {}
281 void CMRegionStack::allocate(size_t size) {
282 _base = NEW_C_HEAP_ARRAY(MemRegion, size);
283 if (_base == NULL)
284 vm_exit_during_initialization("Failed to allocate "
285 "CM region mark stack");
286 _index = 0;
287 // QQQQ cast ...
288 _capacity = (jint) size;
289 }
291 CMRegionStack::~CMRegionStack() {
292 if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base);
293 }
295 void CMRegionStack::push_lock_free(MemRegion mr) {
296 assert(mr.word_size() > 0, "Precondition");
297 while (true) {
298 jint index = _index;
300 if (index >= _capacity) {
301 _overflow = true;
302 return;
303 }
304 // Otherwise...
305 jint next_index = index+1;
306 jint res = Atomic::cmpxchg(next_index, &_index, index);
307 if (res == index) {
308 _base[index] = mr;
309 return;
310 }
311 // Otherwise, we need to try again.
312 }
313 }
315 // Lock-free pop of the region stack. Called during the concurrent
316 // marking / remark phases. Should only be called in tandem with
317 // other lock-free pops.
318 MemRegion CMRegionStack::pop_lock_free() {
319 while (true) {
320 jint index = _index;
322 if (index == 0) {
323 return MemRegion();
324 }
325 // Otherwise...
326 jint next_index = index-1;
327 jint res = Atomic::cmpxchg(next_index, &_index, index);
328 if (res == index) {
329 MemRegion mr = _base[next_index];
330 if (mr.start() != NULL) {
331 assert(mr.end() != NULL, "invariant");
332 assert(mr.word_size() > 0, "invariant");
333 return mr;
334 } else {
335 // that entry was invalidated... let's skip it
336 assert(mr.end() == NULL, "invariant");
337 }
338 }
339 // Otherwise, we need to try again.
340 }
341 }
343 #if 0
344 // The routines that manipulate the region stack with a lock are
345 // not currently used. They should be retained, however, as a
346 // diagnostic aid.
348 void CMRegionStack::push_with_lock(MemRegion mr) {
349 assert(mr.word_size() > 0, "Precondition");
350 MutexLockerEx x(CMRegionStack_lock, Mutex::_no_safepoint_check_flag);
352 if (isFull()) {
353 _overflow = true;
354 return;
355 }
357 _base[_index] = mr;
358 _index += 1;
359 }
361 MemRegion CMRegionStack::pop_with_lock() {
362 MutexLockerEx x(CMRegionStack_lock, Mutex::_no_safepoint_check_flag);
364 while (true) {
365 if (_index == 0) {
366 return MemRegion();
367 }
368 _index -= 1;
370 MemRegion mr = _base[_index];
371 if (mr.start() != NULL) {
372 assert(mr.end() != NULL, "invariant");
373 assert(mr.word_size() > 0, "invariant");
374 return mr;
375 } else {
376 // that entry was invalidated... let's skip it
377 assert(mr.end() == NULL, "invariant");
378 }
379 }
380 }
381 #endif
383 bool CMRegionStack::invalidate_entries_into_cset() {
384 bool result = false;
385 G1CollectedHeap* g1h = G1CollectedHeap::heap();
386 for (int i = 0; i < _oops_do_bound; ++i) {
387 MemRegion mr = _base[i];
388 if (mr.start() != NULL) {
389 assert(mr.end() != NULL, "invariant");
390 assert(mr.word_size() > 0, "invariant");
391 HeapRegion* hr = g1h->heap_region_containing(mr.start());
392 assert(hr != NULL, "invariant");
393 if (hr->in_collection_set()) {
394 // The region points into the collection set
395 _base[i] = MemRegion();
396 result = true;
397 }
398 } else {
399 // that entry was invalidated... let's skip it
400 assert(mr.end() == NULL, "invariant");
401 }
402 }
403 return result;
404 }
406 template<class OopClosureClass>
407 bool CMMarkStack::drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after) {
408 assert(!_drain_in_progress || !_drain_in_progress_yields || yield_after
409 || SafepointSynchronize::is_at_safepoint(),
410 "Drain recursion must be yield-safe.");
411 bool res = true;
412 debug_only(_drain_in_progress = true);
413 debug_only(_drain_in_progress_yields = yield_after);
414 while (!isEmpty()) {
415 oop newOop = pop();
416 assert(G1CollectedHeap::heap()->is_in_reserved(newOop), "Bad pop");
417 assert(newOop->is_oop(), "Expected an oop");
418 assert(bm == NULL || bm->isMarked((HeapWord*)newOop),
419 "only grey objects on this stack");
420 // iterate over the oops in this oop, marking and pushing
421 // the ones in CMS generation.
422 newOop->oop_iterate(cl);
423 if (yield_after && _cm->do_yield_check()) {
424 res = false; break;
425 }
426 }
427 debug_only(_drain_in_progress = false);
428 return res;
429 }
431 void CMMarkStack::oops_do(OopClosure* f) {
432 if (_index == 0) return;
433 assert(_oops_do_bound != -1 && _oops_do_bound <= _index,
434 "Bound must be set.");
435 for (int i = 0; i < _oops_do_bound; i++) {
436 f->do_oop(&_base[i]);
437 }
438 _oops_do_bound = -1;
439 }
441 bool ConcurrentMark::not_yet_marked(oop obj) const {
442 return (_g1h->is_obj_ill(obj)
443 || (_g1h->is_in_permanent(obj)
444 && !nextMarkBitMap()->isMarked((HeapWord*)obj)));
445 }
447 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
448 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
449 #endif // _MSC_VER
451 ConcurrentMark::ConcurrentMark(ReservedSpace rs,
452 int max_regions) :
453 _markBitMap1(rs, MinObjAlignment - 1),
454 _markBitMap2(rs, MinObjAlignment - 1),
456 _parallel_marking_threads(0),
457 _sleep_factor(0.0),
458 _marking_task_overhead(1.0),
459 _cleanup_sleep_factor(0.0),
460 _cleanup_task_overhead(1.0),
461 _cleanup_list("Cleanup List"),
462 _region_bm(max_regions, false /* in_resource_area*/),
463 _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >>
464 CardTableModRefBS::card_shift,
465 false /* in_resource_area*/),
466 _prevMarkBitMap(&_markBitMap1),
467 _nextMarkBitMap(&_markBitMap2),
468 _at_least_one_mark_complete(false),
470 _markStack(this),
471 _regionStack(),
472 // _finger set in set_non_marking_state
474 _max_task_num(MAX2(ParallelGCThreads, (size_t)1)),
475 // _active_tasks set in set_non_marking_state
476 // _tasks set inside the constructor
477 _task_queues(new CMTaskQueueSet((int) _max_task_num)),
478 _terminator(ParallelTaskTerminator((int) _max_task_num, _task_queues)),
480 _has_overflown(false),
481 _concurrent(false),
482 _has_aborted(false),
483 _restart_for_overflow(false),
484 _concurrent_marking_in_progress(false),
485 _should_gray_objects(false),
487 // _verbose_level set below
489 _init_times(),
490 _remark_times(), _remark_mark_times(), _remark_weak_ref_times(),
491 _cleanup_times(),
492 _total_counting_time(0.0),
493 _total_rs_scrub_time(0.0),
495 _parallel_workers(NULL)
496 {
497 CMVerboseLevel verbose_level =
498 (CMVerboseLevel) G1MarkingVerboseLevel;
499 if (verbose_level < no_verbose)
500 verbose_level = no_verbose;
501 if (verbose_level > high_verbose)
502 verbose_level = high_verbose;
503 _verbose_level = verbose_level;
505 if (verbose_low())
506 gclog_or_tty->print_cr("[global] init, heap start = "PTR_FORMAT", "
507 "heap end = "PTR_FORMAT, _heap_start, _heap_end);
509 _markStack.allocate(MarkStackSize);
510 _regionStack.allocate(G1MarkRegionStackSize);
512 // Create & start a ConcurrentMark thread.
513 _cmThread = new ConcurrentMarkThread(this);
514 assert(cmThread() != NULL, "CM Thread should have been created");
515 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
517 _g1h = G1CollectedHeap::heap();
518 assert(CGC_lock != NULL, "Where's the CGC_lock?");
519 assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency");
520 assert(_markBitMap2.covers(rs), "_markBitMap2 inconsistency");
522 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
523 satb_qs.set_buffer_size(G1SATBBufferSize);
525 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num);
526 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num);
528 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
529 _active_tasks = _max_task_num;
530 for (int i = 0; i < (int) _max_task_num; ++i) {
531 CMTaskQueue* task_queue = new CMTaskQueue();
532 task_queue->initialize();
533 _task_queues->register_queue(i, task_queue);
535 _tasks[i] = new CMTask(i, this, task_queue, _task_queues);
536 _accum_task_vtime[i] = 0.0;
537 }
539 if (ConcGCThreads > ParallelGCThreads) {
540 vm_exit_during_initialization("Can't have more ConcGCThreads "
541 "than ParallelGCThreads.");
542 }
543 if (ParallelGCThreads == 0) {
544 // if we are not running with any parallel GC threads we will not
545 // spawn any marking threads either
546 _parallel_marking_threads = 0;
547 _sleep_factor = 0.0;
548 _marking_task_overhead = 1.0;
549 } else {
550 if (ConcGCThreads > 0) {
551 // notice that ConcGCThreads overwrites G1MarkingOverheadPercent
552 // if both are set
554 _parallel_marking_threads = ConcGCThreads;
555 _sleep_factor = 0.0;
556 _marking_task_overhead = 1.0;
557 } else if (G1MarkingOverheadPercent > 0) {
558 // we will calculate the number of parallel marking threads
559 // based on a target overhead with respect to the soft real-time
560 // goal
562 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0;
563 double overall_cm_overhead =
564 (double) MaxGCPauseMillis * marking_overhead /
565 (double) GCPauseIntervalMillis;
566 double cpu_ratio = 1.0 / (double) os::processor_count();
567 double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio);
568 double marking_task_overhead =
569 overall_cm_overhead / marking_thread_num *
570 (double) os::processor_count();
571 double sleep_factor =
572 (1.0 - marking_task_overhead) / marking_task_overhead;
574 _parallel_marking_threads = (size_t) marking_thread_num;
575 _sleep_factor = sleep_factor;
576 _marking_task_overhead = marking_task_overhead;
577 } else {
578 _parallel_marking_threads = MAX2((ParallelGCThreads + 2) / 4, (size_t)1);
579 _sleep_factor = 0.0;
580 _marking_task_overhead = 1.0;
581 }
583 if (parallel_marking_threads() > 1)
584 _cleanup_task_overhead = 1.0;
585 else
586 _cleanup_task_overhead = marking_task_overhead();
587 _cleanup_sleep_factor =
588 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
590 #if 0
591 gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads());
592 gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead());
593 gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor());
594 gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead());
595 gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor());
596 #endif
598 guarantee(parallel_marking_threads() > 0, "peace of mind");
599 _parallel_workers = new FlexibleWorkGang("G1 Parallel Marking Threads",
600 (int) _parallel_marking_threads, false, true);
601 if (_parallel_workers == NULL) {
602 vm_exit_during_initialization("Failed necessary allocation.");
603 } else {
604 _parallel_workers->initialize_workers();
605 }
606 }
608 // so that the call below can read a sensible value
609 _heap_start = (HeapWord*) rs.base();
610 set_non_marking_state();
611 }
613 void ConcurrentMark::update_g1_committed(bool force) {
614 // If concurrent marking is not in progress, then we do not need to
615 // update _heap_end. This has a subtle and important
616 // side-effect. Imagine that two evacuation pauses happen between
617 // marking completion and remark. The first one can grow the
618 // heap (hence now the finger is below the heap end). Then, the
619 // second one could unnecessarily push regions on the region
620 // stack. This causes the invariant that the region stack is empty
621 // at the beginning of remark to be false. By ensuring that we do
622 // not observe heap expansions after marking is complete, then we do
623 // not have this problem.
624 if (!concurrent_marking_in_progress() && !force)
625 return;
627 MemRegion committed = _g1h->g1_committed();
628 assert(committed.start() == _heap_start, "start shouldn't change");
629 HeapWord* new_end = committed.end();
630 if (new_end > _heap_end) {
631 // The heap has been expanded.
633 _heap_end = new_end;
634 }
635 // Notice that the heap can also shrink. However, this only happens
636 // during a Full GC (at least currently) and the entire marking
637 // phase will bail out and the task will not be restarted. So, let's
638 // do nothing.
639 }
641 void ConcurrentMark::reset() {
642 // Starting values for these two. This should be called in a STW
643 // phase. CM will be notified of any future g1_committed expansions
644 // will be at the end of evacuation pauses, when tasks are
645 // inactive.
646 MemRegion committed = _g1h->g1_committed();
647 _heap_start = committed.start();
648 _heap_end = committed.end();
650 // Separated the asserts so that we know which one fires.
651 assert(_heap_start != NULL, "heap bounds should look ok");
652 assert(_heap_end != NULL, "heap bounds should look ok");
653 assert(_heap_start < _heap_end, "heap bounds should look ok");
655 // reset all the marking data structures and any necessary flags
656 clear_marking_state();
658 if (verbose_low())
659 gclog_or_tty->print_cr("[global] resetting");
661 // We do reset all of them, since different phases will use
662 // different number of active threads. So, it's easiest to have all
663 // of them ready.
664 for (int i = 0; i < (int) _max_task_num; ++i) {
665 _tasks[i]->reset(_nextMarkBitMap);
666 }
668 // we need this to make sure that the flag is on during the evac
669 // pause with initial mark piggy-backed
670 set_concurrent_marking_in_progress();
671 }
673 void ConcurrentMark::set_phase(size_t active_tasks, bool concurrent) {
674 assert(active_tasks <= _max_task_num, "we should not have more");
676 _active_tasks = active_tasks;
677 // Need to update the three data structures below according to the
678 // number of active threads for this phase.
679 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues);
680 _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
681 _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
683 _concurrent = concurrent;
684 // We propagate this to all tasks, not just the active ones.
685 for (int i = 0; i < (int) _max_task_num; ++i)
686 _tasks[i]->set_concurrent(concurrent);
688 if (concurrent) {
689 set_concurrent_marking_in_progress();
690 } else {
691 // We currently assume that the concurrent flag has been set to
692 // false before we start remark. At this point we should also be
693 // in a STW phase.
694 assert(!concurrent_marking_in_progress(), "invariant");
695 assert(_finger == _heap_end, "only way to get here");
696 update_g1_committed(true);
697 }
698 }
700 void ConcurrentMark::set_non_marking_state() {
701 // We set the global marking state to some default values when we're
702 // not doing marking.
703 clear_marking_state();
704 _active_tasks = 0;
705 clear_concurrent_marking_in_progress();
706 }
708 ConcurrentMark::~ConcurrentMark() {
709 for (int i = 0; i < (int) _max_task_num; ++i) {
710 delete _task_queues->queue(i);
711 delete _tasks[i];
712 }
713 delete _task_queues;
714 FREE_C_HEAP_ARRAY(CMTask*, _max_task_num);
715 }
717 // This closure is used to mark refs into the g1 generation
718 // from external roots in the CMS bit map.
719 // Called at the first checkpoint.
720 //
722 void ConcurrentMark::clearNextBitmap() {
723 G1CollectedHeap* g1h = G1CollectedHeap::heap();
724 G1CollectorPolicy* g1p = g1h->g1_policy();
726 // Make sure that the concurrent mark thread looks to still be in
727 // the current cycle.
728 guarantee(cmThread()->during_cycle(), "invariant");
730 // We are finishing up the current cycle by clearing the next
731 // marking bitmap and getting it ready for the next cycle. During
732 // this time no other cycle can start. So, let's make sure that this
733 // is the case.
734 guarantee(!g1h->mark_in_progress(), "invariant");
736 // clear the mark bitmap (no grey objects to start with).
737 // We need to do this in chunks and offer to yield in between
738 // each chunk.
739 HeapWord* start = _nextMarkBitMap->startWord();
740 HeapWord* end = _nextMarkBitMap->endWord();
741 HeapWord* cur = start;
742 size_t chunkSize = M;
743 while (cur < end) {
744 HeapWord* next = cur + chunkSize;
745 if (next > end)
746 next = end;
747 MemRegion mr(cur,next);
748 _nextMarkBitMap->clearRange(mr);
749 cur = next;
750 do_yield_check();
752 // Repeat the asserts from above. We'll do them as asserts here to
753 // minimize their overhead on the product. However, we'll have
754 // them as guarantees at the beginning / end of the bitmap
755 // clearing to get some checking in the product.
756 assert(cmThread()->during_cycle(), "invariant");
757 assert(!g1h->mark_in_progress(), "invariant");
758 }
760 // Repeat the asserts from above.
761 guarantee(cmThread()->during_cycle(), "invariant");
762 guarantee(!g1h->mark_in_progress(), "invariant");
763 }
765 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
766 public:
767 bool doHeapRegion(HeapRegion* r) {
768 if (!r->continuesHumongous()) {
769 r->note_start_of_marking(true);
770 }
771 return false;
772 }
773 };
775 void ConcurrentMark::checkpointRootsInitialPre() {
776 G1CollectedHeap* g1h = G1CollectedHeap::heap();
777 G1CollectorPolicy* g1p = g1h->g1_policy();
779 _has_aborted = false;
781 #ifndef PRODUCT
782 if (G1PrintReachableAtInitialMark) {
783 print_reachable("at-cycle-start",
784 true /* use_prev_marking */, true /* all */);
785 }
786 #endif
788 // Initialise marking structures. This has to be done in a STW phase.
789 reset();
790 }
792 class CMMarkRootsClosure: public OopsInGenClosure {
793 private:
794 ConcurrentMark* _cm;
795 G1CollectedHeap* _g1h;
796 bool _do_barrier;
798 public:
799 CMMarkRootsClosure(ConcurrentMark* cm,
800 G1CollectedHeap* g1h,
801 bool do_barrier) : _cm(cm), _g1h(g1h),
802 _do_barrier(do_barrier) { }
804 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
805 virtual void do_oop( oop* p) { do_oop_work(p); }
807 template <class T> void do_oop_work(T* p) {
808 T heap_oop = oopDesc::load_heap_oop(p);
809 if (!oopDesc::is_null(heap_oop)) {
810 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
811 assert(obj->is_oop() || obj->mark() == NULL,
812 "expected an oop, possibly with mark word displaced");
813 HeapWord* addr = (HeapWord*)obj;
814 if (_g1h->is_in_g1_reserved(addr)) {
815 _cm->grayRoot(obj);
816 }
817 }
818 if (_do_barrier) {
819 assert(!_g1h->is_in_g1_reserved(p),
820 "Should be called on external roots");
821 do_barrier(p);
822 }
823 }
824 };
826 void ConcurrentMark::checkpointRootsInitialPost() {
827 G1CollectedHeap* g1h = G1CollectedHeap::heap();
829 // For each region note start of marking.
830 NoteStartOfMarkHRClosure startcl;
831 g1h->heap_region_iterate(&startcl);
833 // Start weak-reference discovery.
834 ReferenceProcessor* rp = g1h->ref_processor();
835 rp->verify_no_references_recorded();
836 rp->enable_discovery(); // enable ("weak") refs discovery
837 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
839 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
840 // This is the start of the marking cycle, we're expected all
841 // threads to have SATB queues with active set to false.
842 satb_mq_set.set_active_all_threads(true, /* new active value */
843 false /* expected_active */);
845 // update_g1_committed() will be called at the end of an evac pause
846 // when marking is on. So, it's also called at the end of the
847 // initial-mark pause to update the heap end, if the heap expands
848 // during it. No need to call it here.
849 }
851 // Checkpoint the roots into this generation from outside
852 // this generation. [Note this initial checkpoint need only
853 // be approximate -- we'll do a catch up phase subsequently.]
854 void ConcurrentMark::checkpointRootsInitial() {
855 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
856 G1CollectedHeap* g1h = G1CollectedHeap::heap();
858 double start = os::elapsedTime();
860 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
861 g1p->record_concurrent_mark_init_start();
862 checkpointRootsInitialPre();
864 // YSR: when concurrent precleaning is in place, we'll
865 // need to clear the cached card table here
867 ResourceMark rm;
868 HandleMark hm;
870 g1h->ensure_parsability(false);
871 g1h->perm_gen()->save_marks();
873 CMMarkRootsClosure notOlder(this, g1h, false);
874 CMMarkRootsClosure older(this, g1h, true);
876 g1h->set_marking_started();
877 g1h->rem_set()->prepare_for_younger_refs_iterate(false);
879 g1h->process_strong_roots(true, // activate StrongRootsScope
880 false, // fake perm gen collection
881 SharedHeap::SO_AllClasses,
882 ¬Older, // Regular roots
883 NULL, // do not visit active blobs
884 &older // Perm Gen Roots
885 );
886 checkpointRootsInitialPost();
888 // Statistics.
889 double end = os::elapsedTime();
890 _init_times.add((end - start) * 1000.0);
892 g1p->record_concurrent_mark_init_end();
893 }
895 /*
896 Notice that in the next two methods, we actually leave the STS
897 during the barrier sync and join it immediately afterwards. If we
898 do not do this, this then the following deadlock can occur: one
899 thread could be in the barrier sync code, waiting for the other
900 thread to also sync up, whereas another one could be trying to
901 yield, while also waiting for the other threads to sync up too.
903 Because the thread that does the sync barrier has left the STS, it
904 is possible to be suspended for a Full GC or an evacuation pause
905 could occur. This is actually safe, since the entering the sync
906 barrier is one of the last things do_marking_step() does, and it
907 doesn't manipulate any data structures afterwards.
908 */
910 void ConcurrentMark::enter_first_sync_barrier(int task_num) {
911 if (verbose_low())
912 gclog_or_tty->print_cr("[%d] entering first barrier", task_num);
914 ConcurrentGCThread::stsLeave();
915 _first_overflow_barrier_sync.enter();
916 ConcurrentGCThread::stsJoin();
917 // at this point everyone should have synced up and not be doing any
918 // more work
920 if (verbose_low())
921 gclog_or_tty->print_cr("[%d] leaving first barrier", task_num);
923 // let task 0 do this
924 if (task_num == 0) {
925 // task 0 is responsible for clearing the global data structures
926 clear_marking_state();
928 if (PrintGC) {
929 gclog_or_tty->date_stamp(PrintGCDateStamps);
930 gclog_or_tty->stamp(PrintGCTimeStamps);
931 gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]");
932 }
933 }
935 // after this, each task should reset its own data structures then
936 // then go into the second barrier
937 }
939 void ConcurrentMark::enter_second_sync_barrier(int task_num) {
940 if (verbose_low())
941 gclog_or_tty->print_cr("[%d] entering second barrier", task_num);
943 ConcurrentGCThread::stsLeave();
944 _second_overflow_barrier_sync.enter();
945 ConcurrentGCThread::stsJoin();
946 // at this point everything should be re-initialised and ready to go
948 if (verbose_low())
949 gclog_or_tty->print_cr("[%d] leaving second barrier", task_num);
950 }
952 void ConcurrentMark::grayRoot(oop p) {
953 HeapWord* addr = (HeapWord*) p;
954 // We can't really check against _heap_start and _heap_end, since it
955 // is possible during an evacuation pause with piggy-backed
956 // initial-mark that the committed space is expanded during the
957 // pause without CM observing this change. So the assertions below
958 // is a bit conservative; but better than nothing.
959 assert(_g1h->g1_committed().contains(addr),
960 "address should be within the heap bounds");
962 if (!_nextMarkBitMap->isMarked(addr))
963 _nextMarkBitMap->parMark(addr);
964 }
966 void ConcurrentMark::grayRegionIfNecessary(MemRegion mr) {
967 // The objects on the region have already been marked "in bulk" by
968 // the caller. We only need to decide whether to push the region on
969 // the region stack or not.
971 if (!concurrent_marking_in_progress() || !_should_gray_objects)
972 // We're done with marking and waiting for remark. We do not need to
973 // push anything else on the region stack.
974 return;
976 HeapWord* finger = _finger;
978 if (verbose_low())
979 gclog_or_tty->print_cr("[global] attempting to push "
980 "region ["PTR_FORMAT", "PTR_FORMAT"), finger is at "
981 PTR_FORMAT, mr.start(), mr.end(), finger);
983 if (mr.start() < finger) {
984 // The finger is always heap region aligned and it is not possible
985 // for mr to span heap regions.
986 assert(mr.end() <= finger, "invariant");
988 // Separated the asserts so that we know which one fires.
989 assert(mr.start() <= mr.end(),
990 "region boundaries should fall within the committed space");
991 assert(_heap_start <= mr.start(),
992 "region boundaries should fall within the committed space");
993 assert(mr.end() <= _heap_end,
994 "region boundaries should fall within the committed space");
995 if (verbose_low())
996 gclog_or_tty->print_cr("[global] region ["PTR_FORMAT", "PTR_FORMAT") "
997 "below the finger, pushing it",
998 mr.start(), mr.end());
1000 if (!region_stack_push_lock_free(mr)) {
1001 if (verbose_low())
1002 gclog_or_tty->print_cr("[global] region stack has overflown.");
1003 }
1004 }
1005 }
1007 void ConcurrentMark::markAndGrayObjectIfNecessary(oop p) {
1008 // The object is not marked by the caller. We need to at least mark
1009 // it and maybe push in on the stack.
1011 HeapWord* addr = (HeapWord*)p;
1012 if (!_nextMarkBitMap->isMarked(addr)) {
1013 // We definitely need to mark it, irrespective whether we bail out
1014 // because we're done with marking.
1015 if (_nextMarkBitMap->parMark(addr)) {
1016 if (!concurrent_marking_in_progress() || !_should_gray_objects)
1017 // If we're done with concurrent marking and we're waiting for
1018 // remark, then we're not pushing anything on the stack.
1019 return;
1021 // No OrderAccess:store_load() is needed. It is implicit in the
1022 // CAS done in parMark(addr) above
1023 HeapWord* finger = _finger;
1025 if (addr < finger) {
1026 if (!mark_stack_push(oop(addr))) {
1027 if (verbose_low())
1028 gclog_or_tty->print_cr("[global] global stack overflow "
1029 "during parMark");
1030 }
1031 }
1032 }
1033 }
1034 }
1036 class CMConcurrentMarkingTask: public AbstractGangTask {
1037 private:
1038 ConcurrentMark* _cm;
1039 ConcurrentMarkThread* _cmt;
1041 public:
1042 void work(int worker_i) {
1043 assert(Thread::current()->is_ConcurrentGC_thread(),
1044 "this should only be done by a conc GC thread");
1045 ResourceMark rm;
1047 double start_vtime = os::elapsedVTime();
1049 ConcurrentGCThread::stsJoin();
1051 assert((size_t) worker_i < _cm->active_tasks(), "invariant");
1052 CMTask* the_task = _cm->task(worker_i);
1053 the_task->record_start_time();
1054 if (!_cm->has_aborted()) {
1055 do {
1056 double start_vtime_sec = os::elapsedVTime();
1057 double start_time_sec = os::elapsedTime();
1058 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
1060 the_task->do_marking_step(mark_step_duration_ms,
1061 true /* do_stealing */,
1062 true /* do_termination */);
1064 double end_time_sec = os::elapsedTime();
1065 double end_vtime_sec = os::elapsedVTime();
1066 double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec;
1067 double elapsed_time_sec = end_time_sec - start_time_sec;
1068 _cm->clear_has_overflown();
1070 bool ret = _cm->do_yield_check(worker_i);
1072 jlong sleep_time_ms;
1073 if (!_cm->has_aborted() && the_task->has_aborted()) {
1074 sleep_time_ms =
1075 (jlong) (elapsed_vtime_sec * _cm->sleep_factor() * 1000.0);
1076 ConcurrentGCThread::stsLeave();
1077 os::sleep(Thread::current(), sleep_time_ms, false);
1078 ConcurrentGCThread::stsJoin();
1079 }
1080 double end_time2_sec = os::elapsedTime();
1081 double elapsed_time2_sec = end_time2_sec - start_time_sec;
1083 #if 0
1084 gclog_or_tty->print_cr("CM: elapsed %1.4lf ms, sleep %1.4lf ms, "
1085 "overhead %1.4lf",
1086 elapsed_vtime_sec * 1000.0, (double) sleep_time_ms,
1087 the_task->conc_overhead(os::elapsedTime()) * 8.0);
1088 gclog_or_tty->print_cr("elapsed time %1.4lf ms, time 2: %1.4lf ms",
1089 elapsed_time_sec * 1000.0, elapsed_time2_sec * 1000.0);
1090 #endif
1091 } while (!_cm->has_aborted() && the_task->has_aborted());
1092 }
1093 the_task->record_end_time();
1094 guarantee(!the_task->has_aborted() || _cm->has_aborted(), "invariant");
1096 ConcurrentGCThread::stsLeave();
1098 double end_vtime = os::elapsedVTime();
1099 _cm->update_accum_task_vtime(worker_i, end_vtime - start_vtime);
1100 }
1102 CMConcurrentMarkingTask(ConcurrentMark* cm,
1103 ConcurrentMarkThread* cmt) :
1104 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { }
1106 ~CMConcurrentMarkingTask() { }
1107 };
1109 void ConcurrentMark::markFromRoots() {
1110 // we might be tempted to assert that:
1111 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1112 // "inconsistent argument?");
1113 // However that wouldn't be right, because it's possible that
1114 // a safepoint is indeed in progress as a younger generation
1115 // stop-the-world GC happens even as we mark in this generation.
1117 _restart_for_overflow = false;
1119 size_t active_workers = MAX2((size_t) 1, parallel_marking_threads());
1120 set_phase(active_workers, true /* concurrent */);
1122 CMConcurrentMarkingTask markingTask(this, cmThread());
1123 if (parallel_marking_threads() > 0)
1124 _parallel_workers->run_task(&markingTask);
1125 else
1126 markingTask.work(0);
1127 print_stats();
1128 }
1130 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1131 // world is stopped at this checkpoint
1132 assert(SafepointSynchronize::is_at_safepoint(),
1133 "world should be stopped");
1134 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1136 // If a full collection has happened, we shouldn't do this.
1137 if (has_aborted()) {
1138 g1h->set_marking_complete(); // So bitmap clearing isn't confused
1139 return;
1140 }
1142 SvcGCMarker sgcm(SvcGCMarker::OTHER);
1144 if (VerifyDuringGC) {
1145 HandleMark hm; // handle scope
1146 gclog_or_tty->print(" VerifyDuringGC:(before)");
1147 Universe::heap()->prepare_for_verify();
1148 Universe::verify(true, false, true);
1149 }
1151 G1CollectorPolicy* g1p = g1h->g1_policy();
1152 g1p->record_concurrent_mark_remark_start();
1154 double start = os::elapsedTime();
1156 checkpointRootsFinalWork();
1158 double mark_work_end = os::elapsedTime();
1160 weakRefsWork(clear_all_soft_refs);
1162 if (has_overflown()) {
1163 // Oops. We overflowed. Restart concurrent marking.
1164 _restart_for_overflow = true;
1165 // Clear the flag. We do not need it any more.
1166 clear_has_overflown();
1167 if (G1TraceMarkStackOverflow)
1168 gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
1169 } else {
1170 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1171 // We're done with marking.
1172 // This is the end of the marking cycle, we're expected all
1173 // threads to have SATB queues with active set to true.
1174 satb_mq_set.set_active_all_threads(false, /* new active value */
1175 true /* expected_active */);
1177 if (VerifyDuringGC) {
1178 HandleMark hm; // handle scope
1179 gclog_or_tty->print(" VerifyDuringGC:(after)");
1180 Universe::heap()->prepare_for_verify();
1181 Universe::heap()->verify(/* allow_dirty */ true,
1182 /* silent */ false,
1183 /* use_prev_marking */ false);
1184 }
1185 assert(!restart_for_overflow(), "sanity");
1186 }
1188 // Reset the marking state if marking completed
1189 if (!restart_for_overflow()) {
1190 set_non_marking_state();
1191 }
1193 #if VERIFY_OBJS_PROCESSED
1194 _scan_obj_cl.objs_processed = 0;
1195 ThreadLocalObjQueue::objs_enqueued = 0;
1196 #endif
1198 // Statistics
1199 double now = os::elapsedTime();
1200 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1201 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1202 _remark_times.add((now - start) * 1000.0);
1204 g1p->record_concurrent_mark_remark_end();
1205 }
1208 #define CARD_BM_TEST_MODE 0
1210 class CalcLiveObjectsClosure: public HeapRegionClosure {
1212 CMBitMapRO* _bm;
1213 ConcurrentMark* _cm;
1214 bool _changed;
1215 bool _yield;
1216 size_t _words_done;
1217 size_t _tot_live;
1218 size_t _tot_used;
1219 size_t _regions_done;
1220 double _start_vtime_sec;
1222 BitMap* _region_bm;
1223 BitMap* _card_bm;
1224 intptr_t _bottom_card_num;
1225 bool _final;
1227 void mark_card_num_range(intptr_t start_card_num, intptr_t last_card_num) {
1228 for (intptr_t i = start_card_num; i <= last_card_num; i++) {
1229 #if CARD_BM_TEST_MODE
1230 guarantee(_card_bm->at(i - _bottom_card_num), "Should already be set.");
1231 #else
1232 _card_bm->par_at_put(i - _bottom_card_num, 1);
1233 #endif
1234 }
1235 }
1237 public:
1238 CalcLiveObjectsClosure(bool final,
1239 CMBitMapRO *bm, ConcurrentMark *cm,
1240 BitMap* region_bm, BitMap* card_bm) :
1241 _bm(bm), _cm(cm), _changed(false), _yield(true),
1242 _words_done(0), _tot_live(0), _tot_used(0),
1243 _region_bm(region_bm), _card_bm(card_bm),_final(final),
1244 _regions_done(0), _start_vtime_sec(0.0)
1245 {
1246 _bottom_card_num =
1247 intptr_t(uintptr_t(G1CollectedHeap::heap()->reserved_region().start()) >>
1248 CardTableModRefBS::card_shift);
1249 }
1251 // It takes a region that's not empty (i.e., it has at least one
1252 // live object in it and sets its corresponding bit on the region
1253 // bitmap to 1. If the region is "starts humongous" it will also set
1254 // to 1 the bits on the region bitmap that correspond to its
1255 // associated "continues humongous" regions.
1256 void set_bit_for_region(HeapRegion* hr) {
1257 assert(!hr->continuesHumongous(), "should have filtered those out");
1259 size_t index = hr->hrs_index();
1260 if (!hr->startsHumongous()) {
1261 // Normal (non-humongous) case: just set the bit.
1262 _region_bm->par_at_put((BitMap::idx_t) index, true);
1263 } else {
1264 // Starts humongous case: calculate how many regions are part of
1265 // this humongous region and then set the bit range. It might
1266 // have been a bit more efficient to look at the object that
1267 // spans these humongous regions to calculate their number from
1268 // the object's size. However, it's a good idea to calculate
1269 // this based on the metadata itself, and not the region
1270 // contents, so that this code is not aware of what goes into
1271 // the humongous regions (in case this changes in the future).
1272 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1273 size_t end_index = index + 1;
1274 while (end_index < g1h->n_regions()) {
1275 HeapRegion* chr = g1h->region_at(end_index);
1276 if (!chr->continuesHumongous()) {
1277 break;
1278 }
1279 end_index += 1;
1280 }
1281 _region_bm->par_at_put_range((BitMap::idx_t) index,
1282 (BitMap::idx_t) end_index, true);
1283 }
1284 }
1286 bool doHeapRegion(HeapRegion* hr) {
1287 if (!_final && _regions_done == 0)
1288 _start_vtime_sec = os::elapsedVTime();
1290 if (hr->continuesHumongous()) {
1291 // We will ignore these here and process them when their
1292 // associated "starts humongous" region is processed (see
1293 // set_bit_for_heap_region()). Note that we cannot rely on their
1294 // associated "starts humongous" region to have their bit set to
1295 // 1 since, due to the region chunking in the parallel region
1296 // iteration, a "continues humongous" region might be visited
1297 // before its associated "starts humongous".
1298 return false;
1299 }
1301 HeapWord* nextTop = hr->next_top_at_mark_start();
1302 HeapWord* start = hr->top_at_conc_mark_count();
1303 assert(hr->bottom() <= start && start <= hr->end() &&
1304 hr->bottom() <= nextTop && nextTop <= hr->end() &&
1305 start <= nextTop,
1306 "Preconditions.");
1307 // Otherwise, record the number of word's we'll examine.
1308 size_t words_done = (nextTop - start);
1309 // Find the first marked object at or after "start".
1310 start = _bm->getNextMarkedWordAddress(start, nextTop);
1311 size_t marked_bytes = 0;
1313 // Below, the term "card num" means the result of shifting an address
1314 // by the card shift -- address 0 corresponds to card number 0. One
1315 // must subtract the card num of the bottom of the heap to obtain a
1316 // card table index.
1317 // The first card num of the sequence of live cards currently being
1318 // constructed. -1 ==> no sequence.
1319 intptr_t start_card_num = -1;
1320 // The last card num of the sequence of live cards currently being
1321 // constructed. -1 ==> no sequence.
1322 intptr_t last_card_num = -1;
1324 while (start < nextTop) {
1325 if (_yield && _cm->do_yield_check()) {
1326 // We yielded. It might be for a full collection, in which case
1327 // all bets are off; terminate the traversal.
1328 if (_cm->has_aborted()) {
1329 _changed = false;
1330 return true;
1331 } else {
1332 // Otherwise, it might be a collection pause, and the region
1333 // we're looking at might be in the collection set. We'll
1334 // abandon this region.
1335 return false;
1336 }
1337 }
1338 oop obj = oop(start);
1339 int obj_sz = obj->size();
1340 // The card num of the start of the current object.
1341 intptr_t obj_card_num =
1342 intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
1344 HeapWord* obj_last = start + obj_sz - 1;
1345 intptr_t obj_last_card_num =
1346 intptr_t(uintptr_t(obj_last) >> CardTableModRefBS::card_shift);
1348 if (obj_card_num != last_card_num) {
1349 if (start_card_num == -1) {
1350 assert(last_card_num == -1, "Both or neither.");
1351 start_card_num = obj_card_num;
1352 } else {
1353 assert(last_card_num != -1, "Both or neither.");
1354 assert(obj_card_num >= last_card_num, "Inv");
1355 if ((obj_card_num - last_card_num) > 1) {
1356 // Mark the last run, and start a new one.
1357 mark_card_num_range(start_card_num, last_card_num);
1358 start_card_num = obj_card_num;
1359 }
1360 }
1361 #if CARD_BM_TEST_MODE
1362 /*
1363 gclog_or_tty->print_cr("Setting bits from %d/%d.",
1364 obj_card_num - _bottom_card_num,
1365 obj_last_card_num - _bottom_card_num);
1366 */
1367 for (intptr_t j = obj_card_num; j <= obj_last_card_num; j++) {
1368 _card_bm->par_at_put(j - _bottom_card_num, 1);
1369 }
1370 #endif
1371 }
1372 // In any case, we set the last card num.
1373 last_card_num = obj_last_card_num;
1375 marked_bytes += (size_t)obj_sz * HeapWordSize;
1376 // Find the next marked object after this one.
1377 start = _bm->getNextMarkedWordAddress(start + 1, nextTop);
1378 _changed = true;
1379 }
1380 // Handle the last range, if any.
1381 if (start_card_num != -1)
1382 mark_card_num_range(start_card_num, last_card_num);
1383 if (_final) {
1384 // Mark the allocated-since-marking portion...
1385 HeapWord* tp = hr->top();
1386 if (nextTop < tp) {
1387 start_card_num =
1388 intptr_t(uintptr_t(nextTop) >> CardTableModRefBS::card_shift);
1389 last_card_num =
1390 intptr_t(uintptr_t(tp) >> CardTableModRefBS::card_shift);
1391 mark_card_num_range(start_card_num, last_card_num);
1392 // This definitely means the region has live objects.
1393 set_bit_for_region(hr);
1394 }
1395 }
1397 hr->add_to_marked_bytes(marked_bytes);
1398 // Update the live region bitmap.
1399 if (marked_bytes > 0) {
1400 set_bit_for_region(hr);
1401 }
1402 hr->set_top_at_conc_mark_count(nextTop);
1403 _tot_live += hr->next_live_bytes();
1404 _tot_used += hr->used();
1405 _words_done = words_done;
1407 if (!_final) {
1408 ++_regions_done;
1409 if (_regions_done % 10 == 0) {
1410 double end_vtime_sec = os::elapsedVTime();
1411 double elapsed_vtime_sec = end_vtime_sec - _start_vtime_sec;
1412 if (elapsed_vtime_sec > (10.0 / 1000.0)) {
1413 jlong sleep_time_ms =
1414 (jlong) (elapsed_vtime_sec * _cm->cleanup_sleep_factor() * 1000.0);
1415 os::sleep(Thread::current(), sleep_time_ms, false);
1416 _start_vtime_sec = end_vtime_sec;
1417 }
1418 }
1419 }
1421 return false;
1422 }
1424 bool changed() { return _changed; }
1425 void reset() { _changed = false; _words_done = 0; }
1426 void no_yield() { _yield = false; }
1427 size_t words_done() { return _words_done; }
1428 size_t tot_live() { return _tot_live; }
1429 size_t tot_used() { return _tot_used; }
1430 };
1433 void ConcurrentMark::calcDesiredRegions() {
1434 _region_bm.clear();
1435 _card_bm.clear();
1436 CalcLiveObjectsClosure calccl(false /*final*/,
1437 nextMarkBitMap(), this,
1438 &_region_bm, &_card_bm);
1439 G1CollectedHeap *g1h = G1CollectedHeap::heap();
1440 g1h->heap_region_iterate(&calccl);
1442 do {
1443 calccl.reset();
1444 g1h->heap_region_iterate(&calccl);
1445 } while (calccl.changed());
1446 }
1448 class G1ParFinalCountTask: public AbstractGangTask {
1449 protected:
1450 G1CollectedHeap* _g1h;
1451 CMBitMap* _bm;
1452 size_t _n_workers;
1453 size_t *_live_bytes;
1454 size_t *_used_bytes;
1455 BitMap* _region_bm;
1456 BitMap* _card_bm;
1457 public:
1458 G1ParFinalCountTask(G1CollectedHeap* g1h, CMBitMap* bm,
1459 BitMap* region_bm, BitMap* card_bm) :
1460 AbstractGangTask("G1 final counting"), _g1h(g1h),
1461 _bm(bm), _region_bm(region_bm), _card_bm(card_bm)
1462 {
1463 if (ParallelGCThreads > 0)
1464 _n_workers = _g1h->workers()->total_workers();
1465 else
1466 _n_workers = 1;
1467 _live_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers);
1468 _used_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers);
1469 }
1471 ~G1ParFinalCountTask() {
1472 FREE_C_HEAP_ARRAY(size_t, _live_bytes);
1473 FREE_C_HEAP_ARRAY(size_t, _used_bytes);
1474 }
1476 void work(int i) {
1477 CalcLiveObjectsClosure calccl(true /*final*/,
1478 _bm, _g1h->concurrent_mark(),
1479 _region_bm, _card_bm);
1480 calccl.no_yield();
1481 if (G1CollectedHeap::use_parallel_gc_threads()) {
1482 _g1h->heap_region_par_iterate_chunked(&calccl, i,
1483 HeapRegion::FinalCountClaimValue);
1484 } else {
1485 _g1h->heap_region_iterate(&calccl);
1486 }
1487 assert(calccl.complete(), "Shouldn't have yielded!");
1489 assert((size_t) i < _n_workers, "invariant");
1490 _live_bytes[i] = calccl.tot_live();
1491 _used_bytes[i] = calccl.tot_used();
1492 }
1493 size_t live_bytes() {
1494 size_t live_bytes = 0;
1495 for (size_t i = 0; i < _n_workers; ++i)
1496 live_bytes += _live_bytes[i];
1497 return live_bytes;
1498 }
1499 size_t used_bytes() {
1500 size_t used_bytes = 0;
1501 for (size_t i = 0; i < _n_workers; ++i)
1502 used_bytes += _used_bytes[i];
1503 return used_bytes;
1504 }
1505 };
1507 class G1ParNoteEndTask;
1509 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1510 G1CollectedHeap* _g1;
1511 int _worker_num;
1512 size_t _max_live_bytes;
1513 size_t _regions_claimed;
1514 size_t _freed_bytes;
1515 FreeRegionList _local_cleanup_list;
1516 HumongousRegionSet _humongous_proxy_set;
1517 double _claimed_region_time;
1518 double _max_region_time;
1520 public:
1521 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1522 int worker_num);
1523 size_t freed_bytes() { return _freed_bytes; }
1524 FreeRegionList* local_cleanup_list() {
1525 return &_local_cleanup_list;
1526 }
1527 HumongousRegionSet* humongous_proxy_set() {
1528 return &_humongous_proxy_set;
1529 }
1531 bool doHeapRegion(HeapRegion *r);
1533 size_t max_live_bytes() { return _max_live_bytes; }
1534 size_t regions_claimed() { return _regions_claimed; }
1535 double claimed_region_time_sec() { return _claimed_region_time; }
1536 double max_region_time_sec() { return _max_region_time; }
1537 };
1539 class G1ParNoteEndTask: public AbstractGangTask {
1540 friend class G1NoteEndOfConcMarkClosure;
1542 protected:
1543 G1CollectedHeap* _g1h;
1544 size_t _max_live_bytes;
1545 size_t _freed_bytes;
1546 FreeRegionList* _cleanup_list;
1548 public:
1549 G1ParNoteEndTask(G1CollectedHeap* g1h,
1550 FreeRegionList* cleanup_list) :
1551 AbstractGangTask("G1 note end"), _g1h(g1h),
1552 _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list) { }
1554 void work(int i) {
1555 double start = os::elapsedTime();
1556 G1NoteEndOfConcMarkClosure g1_note_end(_g1h, i);
1557 if (G1CollectedHeap::use_parallel_gc_threads()) {
1558 _g1h->heap_region_par_iterate_chunked(&g1_note_end, i,
1559 HeapRegion::NoteEndClaimValue);
1560 } else {
1561 _g1h->heap_region_iterate(&g1_note_end);
1562 }
1563 assert(g1_note_end.complete(), "Shouldn't have yielded!");
1565 // Now update the lists
1566 _g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(),
1567 NULL /* free_list */,
1568 g1_note_end.humongous_proxy_set(),
1569 true /* par */);
1570 {
1571 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1572 _max_live_bytes += g1_note_end.max_live_bytes();
1573 _freed_bytes += g1_note_end.freed_bytes();
1575 _cleanup_list->add_as_tail(g1_note_end.local_cleanup_list());
1576 assert(g1_note_end.local_cleanup_list()->is_empty(), "post-condition");
1577 }
1578 double end = os::elapsedTime();
1579 if (G1PrintParCleanupStats) {
1580 gclog_or_tty->print(" Worker thread %d [%8.3f..%8.3f = %8.3f ms] "
1581 "claimed %d regions (tot = %8.3f ms, max = %8.3f ms).\n",
1582 i, start, end, (end-start)*1000.0,
1583 g1_note_end.regions_claimed(),
1584 g1_note_end.claimed_region_time_sec()*1000.0,
1585 g1_note_end.max_region_time_sec()*1000.0);
1586 }
1587 }
1588 size_t max_live_bytes() { return _max_live_bytes; }
1589 size_t freed_bytes() { return _freed_bytes; }
1590 };
1592 class G1ParScrubRemSetTask: public AbstractGangTask {
1593 protected:
1594 G1RemSet* _g1rs;
1595 BitMap* _region_bm;
1596 BitMap* _card_bm;
1597 public:
1598 G1ParScrubRemSetTask(G1CollectedHeap* g1h,
1599 BitMap* region_bm, BitMap* card_bm) :
1600 AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()),
1601 _region_bm(region_bm), _card_bm(card_bm)
1602 {}
1604 void work(int i) {
1605 if (G1CollectedHeap::use_parallel_gc_threads()) {
1606 _g1rs->scrub_par(_region_bm, _card_bm, i,
1607 HeapRegion::ScrubRemSetClaimValue);
1608 } else {
1609 _g1rs->scrub(_region_bm, _card_bm);
1610 }
1611 }
1613 };
1615 G1NoteEndOfConcMarkClosure::
1616 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1617 int worker_num)
1618 : _g1(g1), _worker_num(worker_num),
1619 _max_live_bytes(0), _regions_claimed(0),
1620 _freed_bytes(0),
1621 _claimed_region_time(0.0), _max_region_time(0.0),
1622 _local_cleanup_list("Local Cleanup List"),
1623 _humongous_proxy_set("Local Cleanup Humongous Proxy Set") { }
1625 bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *hr) {
1626 // We use a claim value of zero here because all regions
1627 // were claimed with value 1 in the FinalCount task.
1628 hr->reset_gc_time_stamp();
1629 if (!hr->continuesHumongous()) {
1630 double start = os::elapsedTime();
1631 _regions_claimed++;
1632 hr->note_end_of_marking();
1633 _max_live_bytes += hr->max_live_bytes();
1634 _g1->free_region_if_totally_empty(hr,
1635 &_freed_bytes,
1636 &_local_cleanup_list,
1637 &_humongous_proxy_set,
1638 true /* par */);
1639 double region_time = (os::elapsedTime() - start);
1640 _claimed_region_time += region_time;
1641 if (region_time > _max_region_time) _max_region_time = region_time;
1642 }
1643 return false;
1644 }
1646 void ConcurrentMark::cleanup() {
1647 // world is stopped at this checkpoint
1648 assert(SafepointSynchronize::is_at_safepoint(),
1649 "world should be stopped");
1650 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1652 // If a full collection has happened, we shouldn't do this.
1653 if (has_aborted()) {
1654 g1h->set_marking_complete(); // So bitmap clearing isn't confused
1655 return;
1656 }
1658 g1h->verify_region_sets_optional();
1660 if (VerifyDuringGC) {
1661 HandleMark hm; // handle scope
1662 gclog_or_tty->print(" VerifyDuringGC:(before)");
1663 Universe::heap()->prepare_for_verify();
1664 Universe::verify(/* allow dirty */ true,
1665 /* silent */ false,
1666 /* prev marking */ true);
1667 }
1669 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
1670 g1p->record_concurrent_mark_cleanup_start();
1672 double start = os::elapsedTime();
1674 // Do counting once more with the world stopped for good measure.
1675 G1ParFinalCountTask g1_par_count_task(g1h, nextMarkBitMap(),
1676 &_region_bm, &_card_bm);
1677 if (G1CollectedHeap::use_parallel_gc_threads()) {
1678 assert(g1h->check_heap_region_claim_values(
1679 HeapRegion::InitialClaimValue),
1680 "sanity check");
1682 int n_workers = g1h->workers()->total_workers();
1683 g1h->set_par_threads(n_workers);
1684 g1h->workers()->run_task(&g1_par_count_task);
1685 g1h->set_par_threads(0);
1687 assert(g1h->check_heap_region_claim_values(
1688 HeapRegion::FinalCountClaimValue),
1689 "sanity check");
1690 } else {
1691 g1_par_count_task.work(0);
1692 }
1694 size_t known_garbage_bytes =
1695 g1_par_count_task.used_bytes() - g1_par_count_task.live_bytes();
1696 #if 0
1697 gclog_or_tty->print_cr("used %1.2lf, live %1.2lf, garbage %1.2lf",
1698 (double) g1_par_count_task.used_bytes() / (double) (1024 * 1024),
1699 (double) g1_par_count_task.live_bytes() / (double) (1024 * 1024),
1700 (double) known_garbage_bytes / (double) (1024 * 1024));
1701 #endif // 0
1702 g1p->set_known_garbage_bytes(known_garbage_bytes);
1704 size_t start_used_bytes = g1h->used();
1705 _at_least_one_mark_complete = true;
1706 g1h->set_marking_complete();
1708 double count_end = os::elapsedTime();
1709 double this_final_counting_time = (count_end - start);
1710 if (G1PrintParCleanupStats) {
1711 gclog_or_tty->print_cr("Cleanup:");
1712 gclog_or_tty->print_cr(" Finalize counting: %8.3f ms",
1713 this_final_counting_time*1000.0);
1714 }
1715 _total_counting_time += this_final_counting_time;
1717 // Install newly created mark bitMap as "prev".
1718 swapMarkBitMaps();
1720 g1h->reset_gc_time_stamp();
1722 // Note end of marking in all heap regions.
1723 double note_end_start = os::elapsedTime();
1724 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list);
1725 if (G1CollectedHeap::use_parallel_gc_threads()) {
1726 int n_workers = g1h->workers()->total_workers();
1727 g1h->set_par_threads(n_workers);
1728 g1h->workers()->run_task(&g1_par_note_end_task);
1729 g1h->set_par_threads(0);
1731 assert(g1h->check_heap_region_claim_values(HeapRegion::NoteEndClaimValue),
1732 "sanity check");
1733 } else {
1734 g1_par_note_end_task.work(0);
1735 }
1737 if (!cleanup_list_is_empty()) {
1738 // The cleanup list is not empty, so we'll have to process it
1739 // concurrently. Notify anyone else that might be wanting free
1740 // regions that there will be more free regions coming soon.
1741 g1h->set_free_regions_coming();
1742 }
1743 double note_end_end = os::elapsedTime();
1744 if (G1PrintParCleanupStats) {
1745 gclog_or_tty->print_cr(" note end of marking: %8.3f ms.",
1746 (note_end_end - note_end_start)*1000.0);
1747 }
1750 // call below, since it affects the metric by which we sort the heap
1751 // regions.
1752 if (G1ScrubRemSets) {
1753 double rs_scrub_start = os::elapsedTime();
1754 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm);
1755 if (G1CollectedHeap::use_parallel_gc_threads()) {
1756 int n_workers = g1h->workers()->total_workers();
1757 g1h->set_par_threads(n_workers);
1758 g1h->workers()->run_task(&g1_par_scrub_rs_task);
1759 g1h->set_par_threads(0);
1761 assert(g1h->check_heap_region_claim_values(
1762 HeapRegion::ScrubRemSetClaimValue),
1763 "sanity check");
1764 } else {
1765 g1_par_scrub_rs_task.work(0);
1766 }
1768 double rs_scrub_end = os::elapsedTime();
1769 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
1770 _total_rs_scrub_time += this_rs_scrub_time;
1771 }
1773 // this will also free any regions totally full of garbage objects,
1774 // and sort the regions.
1775 g1h->g1_policy()->record_concurrent_mark_cleanup_end(
1776 g1_par_note_end_task.freed_bytes(),
1777 g1_par_note_end_task.max_live_bytes());
1779 // Statistics.
1780 double end = os::elapsedTime();
1781 _cleanup_times.add((end - start) * 1000.0);
1783 // G1CollectedHeap::heap()->print();
1784 // gclog_or_tty->print_cr("HEAP GC TIME STAMP : %d",
1785 // G1CollectedHeap::heap()->get_gc_time_stamp());
1787 if (PrintGC || PrintGCDetails) {
1788 g1h->print_size_transition(gclog_or_tty,
1789 start_used_bytes,
1790 g1h->used(),
1791 g1h->capacity());
1792 }
1794 size_t cleaned_up_bytes = start_used_bytes - g1h->used();
1795 g1p->decrease_known_garbage_bytes(cleaned_up_bytes);
1797 // We need to make this be a "collection" so any collection pause that
1798 // races with it goes around and waits for completeCleanup to finish.
1799 g1h->increment_total_collections();
1801 if (VerifyDuringGC) {
1802 HandleMark hm; // handle scope
1803 gclog_or_tty->print(" VerifyDuringGC:(after)");
1804 Universe::heap()->prepare_for_verify();
1805 Universe::verify(/* allow dirty */ true,
1806 /* silent */ false,
1807 /* prev marking */ true);
1808 }
1810 g1h->verify_region_sets_optional();
1811 }
1813 void ConcurrentMark::completeCleanup() {
1814 if (has_aborted()) return;
1816 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1818 _cleanup_list.verify_optional();
1819 FreeRegionList local_free_list("Local Cleanup List");
1821 if (G1ConcRegionFreeingVerbose) {
1822 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1823 "cleanup list has "SIZE_FORMAT" entries",
1824 _cleanup_list.length());
1825 }
1827 // Noone else should be accessing the _cleanup_list at this point,
1828 // so it's not necessary to take any locks
1829 while (!_cleanup_list.is_empty()) {
1830 HeapRegion* hr = _cleanup_list.remove_head();
1831 assert(hr != NULL, "the list was not empty");
1832 hr->rem_set()->clear();
1833 local_free_list.add_as_tail(hr);
1835 // Instead of adding one region at a time to the secondary_free_list,
1836 // we accumulate them in the local list and move them a few at a
1837 // time. This also cuts down on the number of notify_all() calls
1838 // we do during this process. We'll also append the local list when
1839 // _cleanup_list is empty (which means we just removed the last
1840 // region from the _cleanup_list).
1841 if ((local_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1842 _cleanup_list.is_empty()) {
1843 if (G1ConcRegionFreeingVerbose) {
1844 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1845 "appending "SIZE_FORMAT" entries to the "
1846 "secondary_free_list, clean list still has "
1847 SIZE_FORMAT" entries",
1848 local_free_list.length(),
1849 _cleanup_list.length());
1850 }
1852 {
1853 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1854 g1h->secondary_free_list_add_as_tail(&local_free_list);
1855 SecondaryFreeList_lock->notify_all();
1856 }
1858 if (G1StressConcRegionFreeing) {
1859 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1860 os::sleep(Thread::current(), (jlong) 1, false);
1861 }
1862 }
1863 }
1864 }
1865 assert(local_free_list.is_empty(), "post-condition");
1866 }
1868 // Support closures for reference procssing in G1
1870 bool G1CMIsAliveClosure::do_object_b(oop obj) {
1871 HeapWord* addr = (HeapWord*)obj;
1872 return addr != NULL &&
1873 (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj));
1874 }
1876 class G1CMKeepAliveClosure: public OopClosure {
1877 G1CollectedHeap* _g1;
1878 ConcurrentMark* _cm;
1879 CMBitMap* _bitMap;
1880 public:
1881 G1CMKeepAliveClosure(G1CollectedHeap* g1, ConcurrentMark* cm,
1882 CMBitMap* bitMap) :
1883 _g1(g1), _cm(cm),
1884 _bitMap(bitMap) {}
1886 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
1887 virtual void do_oop( oop* p) { do_oop_work(p); }
1889 template <class T> void do_oop_work(T* p) {
1890 oop obj = oopDesc::load_decode_heap_oop(p);
1891 HeapWord* addr = (HeapWord*)obj;
1893 if (_cm->verbose_high())
1894 gclog_or_tty->print_cr("\t[0] we're looking at location "
1895 "*"PTR_FORMAT" = "PTR_FORMAT,
1896 p, (void*) obj);
1898 if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(obj)) {
1899 _bitMap->mark(addr);
1900 _cm->mark_stack_push(obj);
1901 }
1902 }
1903 };
1905 class G1CMDrainMarkingStackClosure: public VoidClosure {
1906 CMMarkStack* _markStack;
1907 CMBitMap* _bitMap;
1908 G1CMKeepAliveClosure* _oopClosure;
1909 public:
1910 G1CMDrainMarkingStackClosure(CMBitMap* bitMap, CMMarkStack* markStack,
1911 G1CMKeepAliveClosure* oopClosure) :
1912 _bitMap(bitMap),
1913 _markStack(markStack),
1914 _oopClosure(oopClosure)
1915 {}
1917 void do_void() {
1918 _markStack->drain((OopClosure*)_oopClosure, _bitMap, false);
1919 }
1920 };
1922 // 'Keep Alive' closure used by parallel reference processing.
1923 // An instance of this closure is used in the parallel reference processing
1924 // code rather than an instance of G1CMKeepAliveClosure. We could have used
1925 // the G1CMKeepAliveClosure as it is MT-safe. Also reference objects are
1926 // placed on to discovered ref lists once so we can mark and push with no
1927 // need to check whether the object has already been marked. Using the
1928 // G1CMKeepAliveClosure would mean, however, having all the worker threads
1929 // operating on the global mark stack. This means that an individual
1930 // worker would be doing lock-free pushes while it processes its own
1931 // discovered ref list followed by drain call. If the discovered ref lists
1932 // are unbalanced then this could cause interference with the other
1933 // workers. Using a CMTask (and its embedded local data structures)
1934 // avoids that potential interference.
1935 class G1CMParKeepAliveAndDrainClosure: public OopClosure {
1936 ConcurrentMark* _cm;
1937 CMTask* _task;
1938 CMBitMap* _bitMap;
1939 int _ref_counter_limit;
1940 int _ref_counter;
1941 public:
1942 G1CMParKeepAliveAndDrainClosure(ConcurrentMark* cm,
1943 CMTask* task,
1944 CMBitMap* bitMap) :
1945 _cm(cm), _task(task), _bitMap(bitMap),
1946 _ref_counter_limit(G1RefProcDrainInterval)
1947 {
1948 assert(_ref_counter_limit > 0, "sanity");
1949 _ref_counter = _ref_counter_limit;
1950 }
1952 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
1953 virtual void do_oop( oop* p) { do_oop_work(p); }
1955 template <class T> void do_oop_work(T* p) {
1956 if (!_cm->has_overflown()) {
1957 oop obj = oopDesc::load_decode_heap_oop(p);
1958 if (_cm->verbose_high())
1959 gclog_or_tty->print_cr("\t[%d] we're looking at location "
1960 "*"PTR_FORMAT" = "PTR_FORMAT,
1961 _task->task_id(), p, (void*) obj);
1963 _task->deal_with_reference(obj);
1964 _ref_counter--;
1966 if (_ref_counter == 0) {
1967 // We have dealt with _ref_counter_limit references, pushing them and objects
1968 // reachable from them on to the local stack (and possibly the global stack).
1969 // Call do_marking_step() to process these entries. We call the routine in a
1970 // loop, which we'll exit if there's nothing more to do (i.e. we're done
1971 // with the entries that we've pushed as a result of the deal_with_reference
1972 // calls above) or we overflow.
1973 // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() flag
1974 // while there may still be some work to do. (See the comment at the
1975 // beginning of CMTask::do_marking_step() for those conditions - one of which
1976 // is reaching the specified time target.) It is only when
1977 // CMTask::do_marking_step() returns without setting the has_aborted() flag
1978 // that the marking has completed.
1979 do {
1980 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
1981 _task->do_marking_step(mark_step_duration_ms,
1982 false /* do_stealing */,
1983 false /* do_termination */);
1984 } while (_task->has_aborted() && !_cm->has_overflown());
1985 _ref_counter = _ref_counter_limit;
1986 }
1987 } else {
1988 if (_cm->verbose_high())
1989 gclog_or_tty->print_cr("\t[%d] CM Overflow", _task->task_id());
1990 }
1991 }
1992 };
1994 class G1CMParDrainMarkingStackClosure: public VoidClosure {
1995 ConcurrentMark* _cm;
1996 CMTask* _task;
1997 public:
1998 G1CMParDrainMarkingStackClosure(ConcurrentMark* cm, CMTask* task) :
1999 _cm(cm), _task(task)
2000 {}
2002 void do_void() {
2003 do {
2004 if (_cm->verbose_high())
2005 gclog_or_tty->print_cr("\t[%d] Drain: Calling do marking_step", _task->task_id());
2007 // We call CMTask::do_marking_step() to completely drain the local and
2008 // global marking stacks. The routine is called in a loop, which we'll
2009 // exit if there's nothing more to do (i.e. we'completely drained the
2010 // entries that were pushed as a result of applying the
2011 // G1CMParKeepAliveAndDrainClosure to the entries on the discovered ref
2012 // lists above) or we overflow the global marking stack.
2013 // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() flag
2014 // while there may still be some work to do. (See the comment at the
2015 // beginning of CMTask::do_marking_step() for those conditions - one of which
2016 // is reaching the specified time target.) It is only when
2017 // CMTask::do_marking_step() returns without setting the has_aborted() flag
2018 // that the marking has completed.
2020 _task->do_marking_step(1000000000.0 /* something very large */,
2021 true /* do_stealing */,
2022 true /* do_termination */);
2023 } while (_task->has_aborted() && !_cm->has_overflown());
2024 }
2025 };
2027 // Implementation of AbstractRefProcTaskExecutor for G1
2028 class G1RefProcTaskExecutor: public AbstractRefProcTaskExecutor {
2029 private:
2030 G1CollectedHeap* _g1h;
2031 ConcurrentMark* _cm;
2032 CMBitMap* _bitmap;
2033 WorkGang* _workers;
2034 int _active_workers;
2036 public:
2037 G1RefProcTaskExecutor(G1CollectedHeap* g1h,
2038 ConcurrentMark* cm,
2039 CMBitMap* bitmap,
2040 WorkGang* workers,
2041 int n_workers) :
2042 _g1h(g1h), _cm(cm), _bitmap(bitmap),
2043 _workers(workers), _active_workers(n_workers)
2044 { }
2046 // Executes the given task using concurrent marking worker threads.
2047 virtual void execute(ProcessTask& task);
2048 virtual void execute(EnqueueTask& task);
2049 };
2051 class G1RefProcTaskProxy: public AbstractGangTask {
2052 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
2053 ProcessTask& _proc_task;
2054 G1CollectedHeap* _g1h;
2055 ConcurrentMark* _cm;
2056 CMBitMap* _bitmap;
2058 public:
2059 G1RefProcTaskProxy(ProcessTask& proc_task,
2060 G1CollectedHeap* g1h,
2061 ConcurrentMark* cm,
2062 CMBitMap* bitmap) :
2063 AbstractGangTask("Process reference objects in parallel"),
2064 _proc_task(proc_task), _g1h(g1h), _cm(cm), _bitmap(bitmap)
2065 {}
2067 virtual void work(int i) {
2068 CMTask* marking_task = _cm->task(i);
2069 G1CMIsAliveClosure g1_is_alive(_g1h);
2070 G1CMParKeepAliveAndDrainClosure g1_par_keep_alive(_cm, marking_task, _bitmap);
2071 G1CMParDrainMarkingStackClosure g1_par_drain(_cm, marking_task);
2073 _proc_task.work(i, g1_is_alive, g1_par_keep_alive, g1_par_drain);
2074 }
2075 };
2077 void G1RefProcTaskExecutor::execute(ProcessTask& proc_task) {
2078 assert(_workers != NULL, "Need parallel worker threads.");
2080 G1RefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm, _bitmap);
2082 // We need to reset the phase for each task execution so that
2083 // the termination protocol of CMTask::do_marking_step works.
2084 _cm->set_phase(_active_workers, false /* concurrent */);
2085 _g1h->set_par_threads(_active_workers);
2086 _workers->run_task(&proc_task_proxy);
2087 _g1h->set_par_threads(0);
2088 }
2090 class G1RefEnqueueTaskProxy: public AbstractGangTask {
2091 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
2092 EnqueueTask& _enq_task;
2094 public:
2095 G1RefEnqueueTaskProxy(EnqueueTask& enq_task) :
2096 AbstractGangTask("Enqueue reference objects in parallel"),
2097 _enq_task(enq_task)
2098 { }
2100 virtual void work(int i) {
2101 _enq_task.work(i);
2102 }
2103 };
2105 void G1RefProcTaskExecutor::execute(EnqueueTask& enq_task) {
2106 assert(_workers != NULL, "Need parallel worker threads.");
2108 G1RefEnqueueTaskProxy enq_task_proxy(enq_task);
2110 _g1h->set_par_threads(_active_workers);
2111 _workers->run_task(&enq_task_proxy);
2112 _g1h->set_par_threads(0);
2113 }
2115 void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {
2116 ResourceMark rm;
2117 HandleMark hm;
2118 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2119 ReferenceProcessor* rp = g1h->ref_processor();
2121 // See the comment in G1CollectedHeap::ref_processing_init()
2122 // about how reference processing currently works in G1.
2124 // Process weak references.
2125 rp->setup_policy(clear_all_soft_refs);
2126 assert(_markStack.isEmpty(), "mark stack should be empty");
2128 G1CMIsAliveClosure g1_is_alive(g1h);
2129 G1CMKeepAliveClosure g1_keep_alive(g1h, this, nextMarkBitMap());
2130 G1CMDrainMarkingStackClosure
2131 g1_drain_mark_stack(nextMarkBitMap(), &_markStack, &g1_keep_alive);
2133 // We use the work gang from the G1CollectedHeap and we utilize all
2134 // the worker threads.
2135 int active_workers = MAX2(MIN2(g1h->workers()->total_workers(), (int)_max_task_num), 1);
2137 G1RefProcTaskExecutor par_task_executor(g1h, this, nextMarkBitMap(),
2138 g1h->workers(), active_workers);
2140 if (rp->processing_is_mt()) {
2141 // Set the degree of MT here. If the discovery is done MT, there
2142 // may have been a different number of threads doing the discovery
2143 // and a different number of discovered lists may have Ref objects.
2144 // That is OK as long as the Reference lists are balanced (see
2145 // balance_all_queues() and balance_queues()).
2146 rp->set_mt_degree(active_workers);
2148 rp->process_discovered_references(&g1_is_alive,
2149 &g1_keep_alive,
2150 &g1_drain_mark_stack,
2151 &par_task_executor);
2153 // The work routines of the parallel keep_alive and drain_marking_stack
2154 // will set the has_overflown flag if we overflow the global marking
2155 // stack.
2156 } else {
2157 rp->process_discovered_references(&g1_is_alive,
2158 &g1_keep_alive,
2159 &g1_drain_mark_stack,
2160 NULL);
2162 }
2164 assert(_markStack.overflow() || _markStack.isEmpty(),
2165 "mark stack should be empty (unless it overflowed)");
2166 if (_markStack.overflow()) {
2167 // Should have been done already when we tried to push an
2168 // entry on to the global mark stack. But let's do it again.
2169 set_has_overflown();
2170 }
2172 if (rp->processing_is_mt()) {
2173 assert(rp->num_q() == active_workers, "why not");
2174 rp->enqueue_discovered_references(&par_task_executor);
2175 } else {
2176 rp->enqueue_discovered_references();
2177 }
2179 rp->verify_no_references_recorded();
2180 assert(!rp->discovery_enabled(), "should have been disabled");
2182 // Now clean up stale oops in SymbolTable and StringTable
2183 SymbolTable::unlink(&g1_is_alive);
2184 StringTable::unlink(&g1_is_alive);
2185 }
2187 void ConcurrentMark::swapMarkBitMaps() {
2188 CMBitMapRO* temp = _prevMarkBitMap;
2189 _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap;
2190 _nextMarkBitMap = (CMBitMap*) temp;
2191 }
2193 class CMRemarkTask: public AbstractGangTask {
2194 private:
2195 ConcurrentMark *_cm;
2197 public:
2198 void work(int worker_i) {
2199 // Since all available tasks are actually started, we should
2200 // only proceed if we're supposed to be actived.
2201 if ((size_t)worker_i < _cm->active_tasks()) {
2202 CMTask* task = _cm->task(worker_i);
2203 task->record_start_time();
2204 do {
2205 task->do_marking_step(1000000000.0 /* something very large */,
2206 true /* do_stealing */,
2207 true /* do_termination */);
2208 } while (task->has_aborted() && !_cm->has_overflown());
2209 // If we overflow, then we do not want to restart. We instead
2210 // want to abort remark and do concurrent marking again.
2211 task->record_end_time();
2212 }
2213 }
2215 CMRemarkTask(ConcurrentMark* cm) :
2216 AbstractGangTask("Par Remark"), _cm(cm) { }
2217 };
2219 void ConcurrentMark::checkpointRootsFinalWork() {
2220 ResourceMark rm;
2221 HandleMark hm;
2222 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2224 g1h->ensure_parsability(false);
2226 if (G1CollectedHeap::use_parallel_gc_threads()) {
2227 G1CollectedHeap::StrongRootsScope srs(g1h);
2228 // this is remark, so we'll use up all available threads
2229 int active_workers = ParallelGCThreads;
2230 set_phase(active_workers, false /* concurrent */);
2232 CMRemarkTask remarkTask(this);
2233 // We will start all available threads, even if we decide that the
2234 // active_workers will be fewer. The extra ones will just bail out
2235 // immediately.
2236 int n_workers = g1h->workers()->total_workers();
2237 g1h->set_par_threads(n_workers);
2238 g1h->workers()->run_task(&remarkTask);
2239 g1h->set_par_threads(0);
2240 } else {
2241 G1CollectedHeap::StrongRootsScope srs(g1h);
2242 // this is remark, so we'll use up all available threads
2243 int active_workers = 1;
2244 set_phase(active_workers, false /* concurrent */);
2246 CMRemarkTask remarkTask(this);
2247 // We will start all available threads, even if we decide that the
2248 // active_workers will be fewer. The extra ones will just bail out
2249 // immediately.
2250 remarkTask.work(0);
2251 }
2252 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2253 guarantee(satb_mq_set.completed_buffers_num() == 0, "invariant");
2255 print_stats();
2257 #if VERIFY_OBJS_PROCESSED
2258 if (_scan_obj_cl.objs_processed != ThreadLocalObjQueue::objs_enqueued) {
2259 gclog_or_tty->print_cr("Processed = %d, enqueued = %d.",
2260 _scan_obj_cl.objs_processed,
2261 ThreadLocalObjQueue::objs_enqueued);
2262 guarantee(_scan_obj_cl.objs_processed ==
2263 ThreadLocalObjQueue::objs_enqueued,
2264 "Different number of objs processed and enqueued.");
2265 }
2266 #endif
2267 }
2269 #ifndef PRODUCT
2271 class PrintReachableOopClosure: public OopClosure {
2272 private:
2273 G1CollectedHeap* _g1h;
2274 CMBitMapRO* _bitmap;
2275 outputStream* _out;
2276 bool _use_prev_marking;
2277 bool _all;
2279 public:
2280 PrintReachableOopClosure(CMBitMapRO* bitmap,
2281 outputStream* out,
2282 bool use_prev_marking,
2283 bool all) :
2284 _g1h(G1CollectedHeap::heap()),
2285 _bitmap(bitmap), _out(out), _use_prev_marking(use_prev_marking), _all(all) { }
2287 void do_oop(narrowOop* p) { do_oop_work(p); }
2288 void do_oop( oop* p) { do_oop_work(p); }
2290 template <class T> void do_oop_work(T* p) {
2291 oop obj = oopDesc::load_decode_heap_oop(p);
2292 const char* str = NULL;
2293 const char* str2 = "";
2295 if (obj == NULL) {
2296 str = "";
2297 } else if (!_g1h->is_in_g1_reserved(obj)) {
2298 str = " O";
2299 } else {
2300 HeapRegion* hr = _g1h->heap_region_containing(obj);
2301 guarantee(hr != NULL, "invariant");
2302 bool over_tams = false;
2303 if (_use_prev_marking) {
2304 over_tams = hr->obj_allocated_since_prev_marking(obj);
2305 } else {
2306 over_tams = hr->obj_allocated_since_next_marking(obj);
2307 }
2308 bool marked = _bitmap->isMarked((HeapWord*) obj);
2310 if (over_tams) {
2311 str = " >";
2312 if (marked) {
2313 str2 = " AND MARKED";
2314 }
2315 } else if (marked) {
2316 str = " M";
2317 } else {
2318 str = " NOT";
2319 }
2320 }
2322 _out->print_cr(" "PTR_FORMAT": "PTR_FORMAT"%s%s",
2323 p, (void*) obj, str, str2);
2324 }
2325 };
2327 class PrintReachableObjectClosure : public ObjectClosure {
2328 private:
2329 CMBitMapRO* _bitmap;
2330 outputStream* _out;
2331 bool _use_prev_marking;
2332 bool _all;
2333 HeapRegion* _hr;
2335 public:
2336 PrintReachableObjectClosure(CMBitMapRO* bitmap,
2337 outputStream* out,
2338 bool use_prev_marking,
2339 bool all,
2340 HeapRegion* hr) :
2341 _bitmap(bitmap), _out(out),
2342 _use_prev_marking(use_prev_marking), _all(all), _hr(hr) { }
2344 void do_object(oop o) {
2345 bool over_tams;
2346 if (_use_prev_marking) {
2347 over_tams = _hr->obj_allocated_since_prev_marking(o);
2348 } else {
2349 over_tams = _hr->obj_allocated_since_next_marking(o);
2350 }
2351 bool marked = _bitmap->isMarked((HeapWord*) o);
2352 bool print_it = _all || over_tams || marked;
2354 if (print_it) {
2355 _out->print_cr(" "PTR_FORMAT"%s",
2356 o, (over_tams) ? " >" : (marked) ? " M" : "");
2357 PrintReachableOopClosure oopCl(_bitmap, _out, _use_prev_marking, _all);
2358 o->oop_iterate(&oopCl);
2359 }
2360 }
2361 };
2363 class PrintReachableRegionClosure : public HeapRegionClosure {
2364 private:
2365 CMBitMapRO* _bitmap;
2366 outputStream* _out;
2367 bool _use_prev_marking;
2368 bool _all;
2370 public:
2371 bool doHeapRegion(HeapRegion* hr) {
2372 HeapWord* b = hr->bottom();
2373 HeapWord* e = hr->end();
2374 HeapWord* t = hr->top();
2375 HeapWord* p = NULL;
2376 if (_use_prev_marking) {
2377 p = hr->prev_top_at_mark_start();
2378 } else {
2379 p = hr->next_top_at_mark_start();
2380 }
2381 _out->print_cr("** ["PTR_FORMAT", "PTR_FORMAT"] top: "PTR_FORMAT" "
2382 "TAMS: "PTR_FORMAT, b, e, t, p);
2383 _out->cr();
2385 HeapWord* from = b;
2386 HeapWord* to = t;
2388 if (to > from) {
2389 _out->print_cr("Objects in ["PTR_FORMAT", "PTR_FORMAT"]", from, to);
2390 _out->cr();
2391 PrintReachableObjectClosure ocl(_bitmap, _out,
2392 _use_prev_marking, _all, hr);
2393 hr->object_iterate_mem_careful(MemRegion(from, to), &ocl);
2394 _out->cr();
2395 }
2397 return false;
2398 }
2400 PrintReachableRegionClosure(CMBitMapRO* bitmap,
2401 outputStream* out,
2402 bool use_prev_marking,
2403 bool all) :
2404 _bitmap(bitmap), _out(out), _use_prev_marking(use_prev_marking), _all(all) { }
2405 };
2407 void ConcurrentMark::print_reachable(const char* str,
2408 bool use_prev_marking,
2409 bool all) {
2410 gclog_or_tty->cr();
2411 gclog_or_tty->print_cr("== Doing heap dump... ");
2413 if (G1PrintReachableBaseFile == NULL) {
2414 gclog_or_tty->print_cr(" #### error: no base file defined");
2415 return;
2416 }
2418 if (strlen(G1PrintReachableBaseFile) + 1 + strlen(str) >
2419 (JVM_MAXPATHLEN - 1)) {
2420 gclog_or_tty->print_cr(" #### error: file name too long");
2421 return;
2422 }
2424 char file_name[JVM_MAXPATHLEN];
2425 sprintf(file_name, "%s.%s", G1PrintReachableBaseFile, str);
2426 gclog_or_tty->print_cr(" dumping to file %s", file_name);
2428 fileStream fout(file_name);
2429 if (!fout.is_open()) {
2430 gclog_or_tty->print_cr(" #### error: could not open file");
2431 return;
2432 }
2434 outputStream* out = &fout;
2436 CMBitMapRO* bitmap = NULL;
2437 if (use_prev_marking) {
2438 bitmap = _prevMarkBitMap;
2439 } else {
2440 bitmap = _nextMarkBitMap;
2441 }
2443 out->print_cr("-- USING %s", (use_prev_marking) ? "PTAMS" : "NTAMS");
2444 out->cr();
2446 out->print_cr("--- ITERATING OVER REGIONS");
2447 out->cr();
2448 PrintReachableRegionClosure rcl(bitmap, out, use_prev_marking, all);
2449 _g1h->heap_region_iterate(&rcl);
2450 out->cr();
2452 gclog_or_tty->print_cr(" done");
2453 gclog_or_tty->flush();
2454 }
2456 #endif // PRODUCT
2458 // This note is for drainAllSATBBuffers and the code in between.
2459 // In the future we could reuse a task to do this work during an
2460 // evacuation pause (since now tasks are not active and can be claimed
2461 // during an evacuation pause). This was a late change to the code and
2462 // is currently not being taken advantage of.
2464 class CMGlobalObjectClosure : public ObjectClosure {
2465 private:
2466 ConcurrentMark* _cm;
2468 public:
2469 void do_object(oop obj) {
2470 _cm->deal_with_reference(obj);
2471 }
2473 CMGlobalObjectClosure(ConcurrentMark* cm) : _cm(cm) { }
2474 };
2476 void ConcurrentMark::deal_with_reference(oop obj) {
2477 if (verbose_high())
2478 gclog_or_tty->print_cr("[global] we're dealing with reference "PTR_FORMAT,
2479 (void*) obj);
2482 HeapWord* objAddr = (HeapWord*) obj;
2483 assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
2484 if (_g1h->is_in_g1_reserved(objAddr)) {
2485 assert(obj != NULL, "is_in_g1_reserved should ensure this");
2486 HeapRegion* hr = _g1h->heap_region_containing(obj);
2487 if (_g1h->is_obj_ill(obj, hr)) {
2488 if (verbose_high())
2489 gclog_or_tty->print_cr("[global] "PTR_FORMAT" is not considered "
2490 "marked", (void*) obj);
2492 // we need to mark it first
2493 if (_nextMarkBitMap->parMark(objAddr)) {
2494 // No OrderAccess:store_load() is needed. It is implicit in the
2495 // CAS done in parMark(objAddr) above
2496 HeapWord* finger = _finger;
2497 if (objAddr < finger) {
2498 if (verbose_high())
2499 gclog_or_tty->print_cr("[global] below the global finger "
2500 "("PTR_FORMAT"), pushing it", finger);
2501 if (!mark_stack_push(obj)) {
2502 if (verbose_low())
2503 gclog_or_tty->print_cr("[global] global stack overflow during "
2504 "deal_with_reference");
2505 }
2506 }
2507 }
2508 }
2509 }
2510 }
2512 void ConcurrentMark::drainAllSATBBuffers() {
2513 CMGlobalObjectClosure oc(this);
2514 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2515 satb_mq_set.set_closure(&oc);
2517 while (satb_mq_set.apply_closure_to_completed_buffer()) {
2518 if (verbose_medium())
2519 gclog_or_tty->print_cr("[global] processed an SATB buffer");
2520 }
2522 // no need to check whether we should do this, as this is only
2523 // called during an evacuation pause
2524 satb_mq_set.iterate_closure_all_threads();
2526 satb_mq_set.set_closure(NULL);
2527 assert(satb_mq_set.completed_buffers_num() == 0, "invariant");
2528 }
2530 void ConcurrentMark::markPrev(oop p) {
2531 // Note we are overriding the read-only view of the prev map here, via
2532 // the cast.
2533 ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*)p);
2534 }
2536 void ConcurrentMark::clear(oop p) {
2537 assert(p != NULL && p->is_oop(), "expected an oop");
2538 HeapWord* addr = (HeapWord*)p;
2539 assert(addr >= _nextMarkBitMap->startWord() ||
2540 addr < _nextMarkBitMap->endWord(), "in a region");
2542 _nextMarkBitMap->clear(addr);
2543 }
2545 void ConcurrentMark::clearRangeBothMaps(MemRegion mr) {
2546 // Note we are overriding the read-only view of the prev map here, via
2547 // the cast.
2548 ((CMBitMap*)_prevMarkBitMap)->clearRange(mr);
2549 _nextMarkBitMap->clearRange(mr);
2550 }
2552 HeapRegion*
2553 ConcurrentMark::claim_region(int task_num) {
2554 // "checkpoint" the finger
2555 HeapWord* finger = _finger;
2557 // _heap_end will not change underneath our feet; it only changes at
2558 // yield points.
2559 while (finger < _heap_end) {
2560 assert(_g1h->is_in_g1_reserved(finger), "invariant");
2562 // is the gap between reading the finger and doing the CAS too long?
2564 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
2565 HeapWord* bottom = curr_region->bottom();
2566 HeapWord* end = curr_region->end();
2567 HeapWord* limit = curr_region->next_top_at_mark_start();
2569 if (verbose_low())
2570 gclog_or_tty->print_cr("[%d] curr_region = "PTR_FORMAT" "
2571 "["PTR_FORMAT", "PTR_FORMAT"), "
2572 "limit = "PTR_FORMAT,
2573 task_num, curr_region, bottom, end, limit);
2575 HeapWord* res =
2576 (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger);
2577 if (res == finger) {
2578 // we succeeded
2580 // notice that _finger == end cannot be guaranteed here since,
2581 // someone else might have moved the finger even further
2582 assert(_finger >= end, "the finger should have moved forward");
2584 if (verbose_low())
2585 gclog_or_tty->print_cr("[%d] we were successful with region = "
2586 PTR_FORMAT, task_num, curr_region);
2588 if (limit > bottom) {
2589 if (verbose_low())
2590 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is not empty, "
2591 "returning it ", task_num, curr_region);
2592 return curr_region;
2593 } else {
2594 assert(limit == bottom,
2595 "the region limit should be at bottom");
2596 if (verbose_low())
2597 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is empty, "
2598 "returning NULL", task_num, curr_region);
2599 // we return NULL and the caller should try calling
2600 // claim_region() again.
2601 return NULL;
2602 }
2603 } else {
2604 assert(_finger > finger, "the finger should have moved forward");
2605 if (verbose_low())
2606 gclog_or_tty->print_cr("[%d] somebody else moved the finger, "
2607 "global finger = "PTR_FORMAT", "
2608 "our finger = "PTR_FORMAT,
2609 task_num, _finger, finger);
2611 // read it again
2612 finger = _finger;
2613 }
2614 }
2616 return NULL;
2617 }
2619 bool ConcurrentMark::invalidate_aborted_regions_in_cset() {
2620 bool result = false;
2621 for (int i = 0; i < (int)_max_task_num; ++i) {
2622 CMTask* the_task = _tasks[i];
2623 MemRegion mr = the_task->aborted_region();
2624 if (mr.start() != NULL) {
2625 assert(mr.end() != NULL, "invariant");
2626 assert(mr.word_size() > 0, "invariant");
2627 HeapRegion* hr = _g1h->heap_region_containing(mr.start());
2628 assert(hr != NULL, "invariant");
2629 if (hr->in_collection_set()) {
2630 // The region points into the collection set
2631 the_task->set_aborted_region(MemRegion());
2632 result = true;
2633 }
2634 }
2635 }
2636 return result;
2637 }
2639 bool ConcurrentMark::has_aborted_regions() {
2640 for (int i = 0; i < (int)_max_task_num; ++i) {
2641 CMTask* the_task = _tasks[i];
2642 MemRegion mr = the_task->aborted_region();
2643 if (mr.start() != NULL) {
2644 assert(mr.end() != NULL, "invariant");
2645 assert(mr.word_size() > 0, "invariant");
2646 return true;
2647 }
2648 }
2649 return false;
2650 }
2652 void ConcurrentMark::oops_do(OopClosure* cl) {
2653 if (_markStack.size() > 0 && verbose_low())
2654 gclog_or_tty->print_cr("[global] scanning the global marking stack, "
2655 "size = %d", _markStack.size());
2656 // we first iterate over the contents of the mark stack...
2657 _markStack.oops_do(cl);
2659 for (int i = 0; i < (int)_max_task_num; ++i) {
2660 OopTaskQueue* queue = _task_queues->queue((int)i);
2662 if (queue->size() > 0 && verbose_low())
2663 gclog_or_tty->print_cr("[global] scanning task queue of task %d, "
2664 "size = %d", i, queue->size());
2666 // ...then over the contents of the all the task queues.
2667 queue->oops_do(cl);
2668 }
2670 // Invalidate any entries, that are in the region stack, that
2671 // point into the collection set
2672 if (_regionStack.invalidate_entries_into_cset()) {
2673 // otherwise, any gray objects copied during the evacuation pause
2674 // might not be visited.
2675 assert(_should_gray_objects, "invariant");
2676 }
2678 // Invalidate any aborted regions, recorded in the individual CM
2679 // tasks, that point into the collection set.
2680 if (invalidate_aborted_regions_in_cset()) {
2681 // otherwise, any gray objects copied during the evacuation pause
2682 // might not be visited.
2683 assert(_should_gray_objects, "invariant");
2684 }
2686 }
2688 void ConcurrentMark::clear_marking_state() {
2689 _markStack.setEmpty();
2690 _markStack.clear_overflow();
2691 _regionStack.setEmpty();
2692 _regionStack.clear_overflow();
2693 clear_has_overflown();
2694 _finger = _heap_start;
2696 for (int i = 0; i < (int)_max_task_num; ++i) {
2697 OopTaskQueue* queue = _task_queues->queue(i);
2698 queue->set_empty();
2699 // Clear any partial regions from the CMTasks
2700 _tasks[i]->clear_aborted_region();
2701 }
2702 }
2704 void ConcurrentMark::print_stats() {
2705 if (verbose_stats()) {
2706 gclog_or_tty->print_cr("---------------------------------------------------------------------");
2707 for (size_t i = 0; i < _active_tasks; ++i) {
2708 _tasks[i]->print_stats();
2709 gclog_or_tty->print_cr("---------------------------------------------------------------------");
2710 }
2711 }
2712 }
2714 class CSMarkOopClosure: public OopClosure {
2715 friend class CSMarkBitMapClosure;
2717 G1CollectedHeap* _g1h;
2718 CMBitMap* _bm;
2719 ConcurrentMark* _cm;
2720 oop* _ms;
2721 jint* _array_ind_stack;
2722 int _ms_size;
2723 int _ms_ind;
2724 int _array_increment;
2726 bool push(oop obj, int arr_ind = 0) {
2727 if (_ms_ind == _ms_size) {
2728 gclog_or_tty->print_cr("Mark stack is full.");
2729 return false;
2730 }
2731 _ms[_ms_ind] = obj;
2732 if (obj->is_objArray()) _array_ind_stack[_ms_ind] = arr_ind;
2733 _ms_ind++;
2734 return true;
2735 }
2737 oop pop() {
2738 if (_ms_ind == 0) return NULL;
2739 else {
2740 _ms_ind--;
2741 return _ms[_ms_ind];
2742 }
2743 }
2745 template <class T> bool drain() {
2746 while (_ms_ind > 0) {
2747 oop obj = pop();
2748 assert(obj != NULL, "Since index was non-zero.");
2749 if (obj->is_objArray()) {
2750 jint arr_ind = _array_ind_stack[_ms_ind];
2751 objArrayOop aobj = objArrayOop(obj);
2752 jint len = aobj->length();
2753 jint next_arr_ind = arr_ind + _array_increment;
2754 if (next_arr_ind < len) {
2755 push(obj, next_arr_ind);
2756 }
2757 // Now process this portion of this one.
2758 int lim = MIN2(next_arr_ind, len);
2759 for (int j = arr_ind; j < lim; j++) {
2760 do_oop(aobj->objArrayOopDesc::obj_at_addr<T>(j));
2761 }
2763 } else {
2764 obj->oop_iterate(this);
2765 }
2766 if (abort()) return false;
2767 }
2768 return true;
2769 }
2771 public:
2772 CSMarkOopClosure(ConcurrentMark* cm, int ms_size) :
2773 _g1h(G1CollectedHeap::heap()),
2774 _cm(cm),
2775 _bm(cm->nextMarkBitMap()),
2776 _ms_size(ms_size), _ms_ind(0),
2777 _ms(NEW_C_HEAP_ARRAY(oop, ms_size)),
2778 _array_ind_stack(NEW_C_HEAP_ARRAY(jint, ms_size)),
2779 _array_increment(MAX2(ms_size/8, 16))
2780 {}
2782 ~CSMarkOopClosure() {
2783 FREE_C_HEAP_ARRAY(oop, _ms);
2784 FREE_C_HEAP_ARRAY(jint, _array_ind_stack);
2785 }
2787 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
2788 virtual void do_oop( oop* p) { do_oop_work(p); }
2790 template <class T> void do_oop_work(T* p) {
2791 T heap_oop = oopDesc::load_heap_oop(p);
2792 if (oopDesc::is_null(heap_oop)) return;
2793 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
2794 if (obj->is_forwarded()) {
2795 // If the object has already been forwarded, we have to make sure
2796 // that it's marked. So follow the forwarding pointer. Note that
2797 // this does the right thing for self-forwarding pointers in the
2798 // evacuation failure case.
2799 obj = obj->forwardee();
2800 }
2801 HeapRegion* hr = _g1h->heap_region_containing(obj);
2802 if (hr != NULL) {
2803 if (hr->in_collection_set()) {
2804 if (_g1h->is_obj_ill(obj)) {
2805 _bm->mark((HeapWord*)obj);
2806 if (!push(obj)) {
2807 gclog_or_tty->print_cr("Setting abort in CSMarkOopClosure because push failed.");
2808 set_abort();
2809 }
2810 }
2811 } else {
2812 // Outside the collection set; we need to gray it
2813 _cm->deal_with_reference(obj);
2814 }
2815 }
2816 }
2817 };
2819 class CSMarkBitMapClosure: public BitMapClosure {
2820 G1CollectedHeap* _g1h;
2821 CMBitMap* _bitMap;
2822 ConcurrentMark* _cm;
2823 CSMarkOopClosure _oop_cl;
2824 public:
2825 CSMarkBitMapClosure(ConcurrentMark* cm, int ms_size) :
2826 _g1h(G1CollectedHeap::heap()),
2827 _bitMap(cm->nextMarkBitMap()),
2828 _oop_cl(cm, ms_size)
2829 {}
2831 ~CSMarkBitMapClosure() {}
2833 bool do_bit(size_t offset) {
2834 // convert offset into a HeapWord*
2835 HeapWord* addr = _bitMap->offsetToHeapWord(offset);
2836 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
2837 "address out of range");
2838 assert(_bitMap->isMarked(addr), "tautology");
2839 oop obj = oop(addr);
2840 if (!obj->is_forwarded()) {
2841 if (!_oop_cl.push(obj)) return false;
2842 if (UseCompressedOops) {
2843 if (!_oop_cl.drain<narrowOop>()) return false;
2844 } else {
2845 if (!_oop_cl.drain<oop>()) return false;
2846 }
2847 }
2848 // Otherwise...
2849 return true;
2850 }
2851 };
2854 class CompleteMarkingInCSHRClosure: public HeapRegionClosure {
2855 CMBitMap* _bm;
2856 CSMarkBitMapClosure _bit_cl;
2857 enum SomePrivateConstants {
2858 MSSize = 1000
2859 };
2860 bool _completed;
2861 public:
2862 CompleteMarkingInCSHRClosure(ConcurrentMark* cm) :
2863 _bm(cm->nextMarkBitMap()),
2864 _bit_cl(cm, MSSize),
2865 _completed(true)
2866 {}
2868 ~CompleteMarkingInCSHRClosure() {}
2870 bool doHeapRegion(HeapRegion* r) {
2871 if (!r->evacuation_failed()) {
2872 MemRegion mr = MemRegion(r->bottom(), r->next_top_at_mark_start());
2873 if (!mr.is_empty()) {
2874 if (!_bm->iterate(&_bit_cl, mr)) {
2875 _completed = false;
2876 return true;
2877 }
2878 }
2879 }
2880 return false;
2881 }
2883 bool completed() { return _completed; }
2884 };
2886 class ClearMarksInHRClosure: public HeapRegionClosure {
2887 CMBitMap* _bm;
2888 public:
2889 ClearMarksInHRClosure(CMBitMap* bm): _bm(bm) { }
2891 bool doHeapRegion(HeapRegion* r) {
2892 if (!r->used_region().is_empty() && !r->evacuation_failed()) {
2893 MemRegion usedMR = r->used_region();
2894 _bm->clearRange(r->used_region());
2895 }
2896 return false;
2897 }
2898 };
2900 void ConcurrentMark::complete_marking_in_collection_set() {
2901 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2903 if (!g1h->mark_in_progress()) {
2904 g1h->g1_policy()->record_mark_closure_time(0.0);
2905 return;
2906 }
2908 int i = 1;
2909 double start = os::elapsedTime();
2910 while (true) {
2911 i++;
2912 CompleteMarkingInCSHRClosure cmplt(this);
2913 g1h->collection_set_iterate(&cmplt);
2914 if (cmplt.completed()) break;
2915 }
2916 double end_time = os::elapsedTime();
2917 double elapsed_time_ms = (end_time - start) * 1000.0;
2918 g1h->g1_policy()->record_mark_closure_time(elapsed_time_ms);
2920 ClearMarksInHRClosure clr(nextMarkBitMap());
2921 g1h->collection_set_iterate(&clr);
2922 }
2924 // The next two methods deal with the following optimisation. Some
2925 // objects are gray by being marked and located above the finger. If
2926 // they are copied, during an evacuation pause, below the finger then
2927 // the need to be pushed on the stack. The observation is that, if
2928 // there are no regions in the collection set located above the
2929 // finger, then the above cannot happen, hence we do not need to
2930 // explicitly gray any objects when copying them to below the
2931 // finger. The global stack will be scanned to ensure that, if it
2932 // points to objects being copied, it will update their
2933 // location. There is a tricky situation with the gray objects in
2934 // region stack that are being coped, however. See the comment in
2935 // newCSet().
2937 void ConcurrentMark::newCSet() {
2938 if (!concurrent_marking_in_progress())
2939 // nothing to do if marking is not in progress
2940 return;
2942 // find what the lowest finger is among the global and local fingers
2943 _min_finger = _finger;
2944 for (int i = 0; i < (int)_max_task_num; ++i) {
2945 CMTask* task = _tasks[i];
2946 HeapWord* task_finger = task->finger();
2947 if (task_finger != NULL && task_finger < _min_finger)
2948 _min_finger = task_finger;
2949 }
2951 _should_gray_objects = false;
2953 // This fixes a very subtle and fustrating bug. It might be the case
2954 // that, during en evacuation pause, heap regions that contain
2955 // objects that are gray (by being in regions contained in the
2956 // region stack) are included in the collection set. Since such gray
2957 // objects will be moved, and because it's not easy to redirect
2958 // region stack entries to point to a new location (because objects
2959 // in one region might be scattered to multiple regions after they
2960 // are copied), one option is to ensure that all marked objects
2961 // copied during a pause are pushed on the stack. Notice, however,
2962 // that this problem can only happen when the region stack is not
2963 // empty during an evacuation pause. So, we make the fix a bit less
2964 // conservative and ensure that regions are pushed on the stack,
2965 // irrespective whether all collection set regions are below the
2966 // finger, if the region stack is not empty. This is expected to be
2967 // a rare case, so I don't think it's necessary to be smarted about it.
2968 if (!region_stack_empty() || has_aborted_regions())
2969 _should_gray_objects = true;
2970 }
2972 void ConcurrentMark::registerCSetRegion(HeapRegion* hr) {
2973 if (!concurrent_marking_in_progress())
2974 return;
2976 HeapWord* region_end = hr->end();
2977 if (region_end > _min_finger)
2978 _should_gray_objects = true;
2979 }
2981 // abandon current marking iteration due to a Full GC
2982 void ConcurrentMark::abort() {
2983 // Clear all marks to force marking thread to do nothing
2984 _nextMarkBitMap->clearAll();
2985 // Empty mark stack
2986 clear_marking_state();
2987 for (int i = 0; i < (int)_max_task_num; ++i) {
2988 _tasks[i]->clear_region_fields();
2989 }
2990 _has_aborted = true;
2992 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2993 satb_mq_set.abandon_partial_marking();
2994 // This can be called either during or outside marking, we'll read
2995 // the expected_active value from the SATB queue set.
2996 satb_mq_set.set_active_all_threads(
2997 false, /* new active value */
2998 satb_mq_set.is_active() /* expected_active */);
2999 }
3001 static void print_ms_time_info(const char* prefix, const char* name,
3002 NumberSeq& ns) {
3003 gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
3004 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
3005 if (ns.num() > 0) {
3006 gclog_or_tty->print_cr("%s [std. dev = %8.2f ms, max = %8.2f ms]",
3007 prefix, ns.sd(), ns.maximum());
3008 }
3009 }
3011 void ConcurrentMark::print_summary_info() {
3012 gclog_or_tty->print_cr(" Concurrent marking:");
3013 print_ms_time_info(" ", "init marks", _init_times);
3014 print_ms_time_info(" ", "remarks", _remark_times);
3015 {
3016 print_ms_time_info(" ", "final marks", _remark_mark_times);
3017 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
3019 }
3020 print_ms_time_info(" ", "cleanups", _cleanup_times);
3021 gclog_or_tty->print_cr(" Final counting total time = %8.2f s (avg = %8.2f ms).",
3022 _total_counting_time,
3023 (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 /
3024 (double)_cleanup_times.num()
3025 : 0.0));
3026 if (G1ScrubRemSets) {
3027 gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
3028 _total_rs_scrub_time,
3029 (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 /
3030 (double)_cleanup_times.num()
3031 : 0.0));
3032 }
3033 gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.",
3034 (_init_times.sum() + _remark_times.sum() +
3035 _cleanup_times.sum())/1000.0);
3036 gclog_or_tty->print_cr(" Total concurrent time = %8.2f s "
3037 "(%8.2f s marking, %8.2f s counting).",
3038 cmThread()->vtime_accum(),
3039 cmThread()->vtime_mark_accum(),
3040 cmThread()->vtime_count_accum());
3041 }
3043 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
3044 _parallel_workers->print_worker_threads_on(st);
3045 }
3047 // Closures
3048 // XXX: there seems to be a lot of code duplication here;
3049 // should refactor and consolidate the shared code.
3051 // This closure is used to mark refs into the CMS generation in
3052 // the CMS bit map. Called at the first checkpoint.
3054 // We take a break if someone is trying to stop the world.
3055 bool ConcurrentMark::do_yield_check(int worker_i) {
3056 if (should_yield()) {
3057 if (worker_i == 0)
3058 _g1h->g1_policy()->record_concurrent_pause();
3059 cmThread()->yield();
3060 if (worker_i == 0)
3061 _g1h->g1_policy()->record_concurrent_pause_end();
3062 return true;
3063 } else {
3064 return false;
3065 }
3066 }
3068 bool ConcurrentMark::should_yield() {
3069 return cmThread()->should_yield();
3070 }
3072 bool ConcurrentMark::containing_card_is_marked(void* p) {
3073 size_t offset = pointer_delta(p, _g1h->reserved_region().start(), 1);
3074 return _card_bm.at(offset >> CardTableModRefBS::card_shift);
3075 }
3077 bool ConcurrentMark::containing_cards_are_marked(void* start,
3078 void* last) {
3079 return
3080 containing_card_is_marked(start) &&
3081 containing_card_is_marked(last);
3082 }
3084 #ifndef PRODUCT
3085 // for debugging purposes
3086 void ConcurrentMark::print_finger() {
3087 gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT,
3088 _heap_start, _heap_end, _finger);
3089 for (int i = 0; i < (int) _max_task_num; ++i) {
3090 gclog_or_tty->print(" %d: "PTR_FORMAT, i, _tasks[i]->finger());
3091 }
3092 gclog_or_tty->print_cr("");
3093 }
3094 #endif
3096 // Closure for iteration over bitmaps
3097 class CMBitMapClosure : public BitMapClosure {
3098 private:
3099 // the bitmap that is being iterated over
3100 CMBitMap* _nextMarkBitMap;
3101 ConcurrentMark* _cm;
3102 CMTask* _task;
3103 // true if we're scanning a heap region claimed by the task (so that
3104 // we move the finger along), false if we're not, i.e. currently when
3105 // scanning a heap region popped from the region stack (so that we
3106 // do not move the task finger along; it'd be a mistake if we did so).
3107 bool _scanning_heap_region;
3109 public:
3110 CMBitMapClosure(CMTask *task,
3111 ConcurrentMark* cm,
3112 CMBitMap* nextMarkBitMap)
3113 : _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { }
3115 void set_scanning_heap_region(bool scanning_heap_region) {
3116 _scanning_heap_region = scanning_heap_region;
3117 }
3119 bool do_bit(size_t offset) {
3120 HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset);
3121 assert(_nextMarkBitMap->isMarked(addr), "invariant");
3122 assert( addr < _cm->finger(), "invariant");
3124 if (_scanning_heap_region) {
3125 statsOnly( _task->increase_objs_found_on_bitmap() );
3126 assert(addr >= _task->finger(), "invariant");
3127 // We move that task's local finger along.
3128 _task->move_finger_to(addr);
3129 } else {
3130 // We move the task's region finger along.
3131 _task->move_region_finger_to(addr);
3132 }
3134 _task->scan_object(oop(addr));
3135 // we only partially drain the local queue and global stack
3136 _task->drain_local_queue(true);
3137 _task->drain_global_stack(true);
3139 // if the has_aborted flag has been raised, we need to bail out of
3140 // the iteration
3141 return !_task->has_aborted();
3142 }
3143 };
3145 // Closure for iterating over objects, currently only used for
3146 // processing SATB buffers.
3147 class CMObjectClosure : public ObjectClosure {
3148 private:
3149 CMTask* _task;
3151 public:
3152 void do_object(oop obj) {
3153 _task->deal_with_reference(obj);
3154 }
3156 CMObjectClosure(CMTask* task) : _task(task) { }
3157 };
3159 // Closure for iterating over object fields
3160 class CMOopClosure : public OopClosure {
3161 private:
3162 G1CollectedHeap* _g1h;
3163 ConcurrentMark* _cm;
3164 CMTask* _task;
3166 public:
3167 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
3168 virtual void do_oop( oop* p) { do_oop_work(p); }
3170 template <class T> void do_oop_work(T* p) {
3171 assert( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant");
3172 assert(!_g1h->is_on_free_list(
3173 _g1h->heap_region_containing((HeapWord*) p)), "invariant");
3175 oop obj = oopDesc::load_decode_heap_oop(p);
3176 if (_cm->verbose_high())
3177 gclog_or_tty->print_cr("[%d] we're looking at location "
3178 "*"PTR_FORMAT" = "PTR_FORMAT,
3179 _task->task_id(), p, (void*) obj);
3180 _task->deal_with_reference(obj);
3181 }
3183 CMOopClosure(G1CollectedHeap* g1h,
3184 ConcurrentMark* cm,
3185 CMTask* task)
3186 : _g1h(g1h), _cm(cm), _task(task)
3187 {
3188 _ref_processor = g1h->ref_processor();
3189 assert(_ref_processor != NULL, "should not be NULL");
3190 }
3191 };
3193 void CMTask::setup_for_region(HeapRegion* hr) {
3194 // Separated the asserts so that we know which one fires.
3195 assert(hr != NULL,
3196 "claim_region() should have filtered out continues humongous regions");
3197 assert(!hr->continuesHumongous(),
3198 "claim_region() should have filtered out continues humongous regions");
3200 if (_cm->verbose_low())
3201 gclog_or_tty->print_cr("[%d] setting up for region "PTR_FORMAT,
3202 _task_id, hr);
3204 _curr_region = hr;
3205 _finger = hr->bottom();
3206 update_region_limit();
3207 }
3209 void CMTask::update_region_limit() {
3210 HeapRegion* hr = _curr_region;
3211 HeapWord* bottom = hr->bottom();
3212 HeapWord* limit = hr->next_top_at_mark_start();
3214 if (limit == bottom) {
3215 if (_cm->verbose_low())
3216 gclog_or_tty->print_cr("[%d] found an empty region "
3217 "["PTR_FORMAT", "PTR_FORMAT")",
3218 _task_id, bottom, limit);
3219 // The region was collected underneath our feet.
3220 // We set the finger to bottom to ensure that the bitmap
3221 // iteration that will follow this will not do anything.
3222 // (this is not a condition that holds when we set the region up,
3223 // as the region is not supposed to be empty in the first place)
3224 _finger = bottom;
3225 } else if (limit >= _region_limit) {
3226 assert(limit >= _finger, "peace of mind");
3227 } else {
3228 assert(limit < _region_limit, "only way to get here");
3229 // This can happen under some pretty unusual circumstances. An
3230 // evacuation pause empties the region underneath our feet (NTAMS
3231 // at bottom). We then do some allocation in the region (NTAMS
3232 // stays at bottom), followed by the region being used as a GC
3233 // alloc region (NTAMS will move to top() and the objects
3234 // originally below it will be grayed). All objects now marked in
3235 // the region are explicitly grayed, if below the global finger,
3236 // and we do not need in fact to scan anything else. So, we simply
3237 // set _finger to be limit to ensure that the bitmap iteration
3238 // doesn't do anything.
3239 _finger = limit;
3240 }
3242 _region_limit = limit;
3243 }
3245 void CMTask::giveup_current_region() {
3246 assert(_curr_region != NULL, "invariant");
3247 if (_cm->verbose_low())
3248 gclog_or_tty->print_cr("[%d] giving up region "PTR_FORMAT,
3249 _task_id, _curr_region);
3250 clear_region_fields();
3251 }
3253 void CMTask::clear_region_fields() {
3254 // Values for these three fields that indicate that we're not
3255 // holding on to a region.
3256 _curr_region = NULL;
3257 _finger = NULL;
3258 _region_limit = NULL;
3260 _region_finger = NULL;
3261 }
3263 void CMTask::reset(CMBitMap* nextMarkBitMap) {
3264 guarantee(nextMarkBitMap != NULL, "invariant");
3266 if (_cm->verbose_low())
3267 gclog_or_tty->print_cr("[%d] resetting", _task_id);
3269 _nextMarkBitMap = nextMarkBitMap;
3270 clear_region_fields();
3271 assert(_aborted_region.is_empty(), "should have been cleared");
3273 _calls = 0;
3274 _elapsed_time_ms = 0.0;
3275 _termination_time_ms = 0.0;
3276 _termination_start_time_ms = 0.0;
3278 #if _MARKING_STATS_
3279 _local_pushes = 0;
3280 _local_pops = 0;
3281 _local_max_size = 0;
3282 _objs_scanned = 0;
3283 _global_pushes = 0;
3284 _global_pops = 0;
3285 _global_max_size = 0;
3286 _global_transfers_to = 0;
3287 _global_transfers_from = 0;
3288 _region_stack_pops = 0;
3289 _regions_claimed = 0;
3290 _objs_found_on_bitmap = 0;
3291 _satb_buffers_processed = 0;
3292 _steal_attempts = 0;
3293 _steals = 0;
3294 _aborted = 0;
3295 _aborted_overflow = 0;
3296 _aborted_cm_aborted = 0;
3297 _aborted_yield = 0;
3298 _aborted_timed_out = 0;
3299 _aborted_satb = 0;
3300 _aborted_termination = 0;
3301 #endif // _MARKING_STATS_
3302 }
3304 bool CMTask::should_exit_termination() {
3305 regular_clock_call();
3306 // This is called when we are in the termination protocol. We should
3307 // quit if, for some reason, this task wants to abort or the global
3308 // stack is not empty (this means that we can get work from it).
3309 return !_cm->mark_stack_empty() || has_aborted();
3310 }
3312 // This determines whether the method below will check both the local
3313 // and global fingers when determining whether to push on the stack a
3314 // gray object (value 1) or whether it will only check the global one
3315 // (value 0). The tradeoffs are that the former will be a bit more
3316 // accurate and possibly push less on the stack, but it might also be
3317 // a little bit slower.
3319 #define _CHECK_BOTH_FINGERS_ 1
3321 void CMTask::deal_with_reference(oop obj) {
3322 if (_cm->verbose_high())
3323 gclog_or_tty->print_cr("[%d] we're dealing with reference = "PTR_FORMAT,
3324 _task_id, (void*) obj);
3326 ++_refs_reached;
3328 HeapWord* objAddr = (HeapWord*) obj;
3329 assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
3330 if (_g1h->is_in_g1_reserved(objAddr)) {
3331 assert(obj != NULL, "is_in_g1_reserved should ensure this");
3332 HeapRegion* hr = _g1h->heap_region_containing(obj);
3333 if (_g1h->is_obj_ill(obj, hr)) {
3334 if (_cm->verbose_high())
3335 gclog_or_tty->print_cr("[%d] "PTR_FORMAT" is not considered marked",
3336 _task_id, (void*) obj);
3338 // we need to mark it first
3339 if (_nextMarkBitMap->parMark(objAddr)) {
3340 // No OrderAccess:store_load() is needed. It is implicit in the
3341 // CAS done in parMark(objAddr) above
3342 HeapWord* global_finger = _cm->finger();
3344 #if _CHECK_BOTH_FINGERS_
3345 // we will check both the local and global fingers
3347 if (_finger != NULL && objAddr < _finger) {
3348 if (_cm->verbose_high())
3349 gclog_or_tty->print_cr("[%d] below the local finger ("PTR_FORMAT"), "
3350 "pushing it", _task_id, _finger);
3351 push(obj);
3352 } else if (_curr_region != NULL && objAddr < _region_limit) {
3353 // do nothing
3354 } else if (objAddr < global_finger) {
3355 // Notice that the global finger might be moving forward
3356 // concurrently. This is not a problem. In the worst case, we
3357 // mark the object while it is above the global finger and, by
3358 // the time we read the global finger, it has moved forward
3359 // passed this object. In this case, the object will probably
3360 // be visited when a task is scanning the region and will also
3361 // be pushed on the stack. So, some duplicate work, but no
3362 // correctness problems.
3364 if (_cm->verbose_high())
3365 gclog_or_tty->print_cr("[%d] below the global finger "
3366 "("PTR_FORMAT"), pushing it",
3367 _task_id, global_finger);
3368 push(obj);
3369 } else {
3370 // do nothing
3371 }
3372 #else // _CHECK_BOTH_FINGERS_
3373 // we will only check the global finger
3375 if (objAddr < global_finger) {
3376 // see long comment above
3378 if (_cm->verbose_high())
3379 gclog_or_tty->print_cr("[%d] below the global finger "
3380 "("PTR_FORMAT"), pushing it",
3381 _task_id, global_finger);
3382 push(obj);
3383 }
3384 #endif // _CHECK_BOTH_FINGERS_
3385 }
3386 }
3387 }
3388 }
3390 void CMTask::push(oop obj) {
3391 HeapWord* objAddr = (HeapWord*) obj;
3392 assert(_g1h->is_in_g1_reserved(objAddr), "invariant");
3393 assert(!_g1h->is_on_free_list(
3394 _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant");
3395 assert(!_g1h->is_obj_ill(obj), "invariant");
3396 assert(_nextMarkBitMap->isMarked(objAddr), "invariant");
3398 if (_cm->verbose_high())
3399 gclog_or_tty->print_cr("[%d] pushing "PTR_FORMAT, _task_id, (void*) obj);
3401 if (!_task_queue->push(obj)) {
3402 // The local task queue looks full. We need to push some entries
3403 // to the global stack.
3405 if (_cm->verbose_medium())
3406 gclog_or_tty->print_cr("[%d] task queue overflow, "
3407 "moving entries to the global stack",
3408 _task_id);
3409 move_entries_to_global_stack();
3411 // this should succeed since, even if we overflow the global
3412 // stack, we should have definitely removed some entries from the
3413 // local queue. So, there must be space on it.
3414 bool success = _task_queue->push(obj);
3415 assert(success, "invariant");
3416 }
3418 statsOnly( int tmp_size = _task_queue->size();
3419 if (tmp_size > _local_max_size)
3420 _local_max_size = tmp_size;
3421 ++_local_pushes );
3422 }
3424 void CMTask::reached_limit() {
3425 assert(_words_scanned >= _words_scanned_limit ||
3426 _refs_reached >= _refs_reached_limit ,
3427 "shouldn't have been called otherwise");
3428 regular_clock_call();
3429 }
3431 void CMTask::regular_clock_call() {
3432 if (has_aborted())
3433 return;
3435 // First, we need to recalculate the words scanned and refs reached
3436 // limits for the next clock call.
3437 recalculate_limits();
3439 // During the regular clock call we do the following
3441 // (1) If an overflow has been flagged, then we abort.
3442 if (_cm->has_overflown()) {
3443 set_has_aborted();
3444 return;
3445 }
3447 // If we are not concurrent (i.e. we're doing remark) we don't need
3448 // to check anything else. The other steps are only needed during
3449 // the concurrent marking phase.
3450 if (!concurrent())
3451 return;
3453 // (2) If marking has been aborted for Full GC, then we also abort.
3454 if (_cm->has_aborted()) {
3455 set_has_aborted();
3456 statsOnly( ++_aborted_cm_aborted );
3457 return;
3458 }
3460 double curr_time_ms = os::elapsedVTime() * 1000.0;
3462 // (3) If marking stats are enabled, then we update the step history.
3463 #if _MARKING_STATS_
3464 if (_words_scanned >= _words_scanned_limit)
3465 ++_clock_due_to_scanning;
3466 if (_refs_reached >= _refs_reached_limit)
3467 ++_clock_due_to_marking;
3469 double last_interval_ms = curr_time_ms - _interval_start_time_ms;
3470 _interval_start_time_ms = curr_time_ms;
3471 _all_clock_intervals_ms.add(last_interval_ms);
3473 if (_cm->verbose_medium()) {
3474 gclog_or_tty->print_cr("[%d] regular clock, interval = %1.2lfms, "
3475 "scanned = %d%s, refs reached = %d%s",
3476 _task_id, last_interval_ms,
3477 _words_scanned,
3478 (_words_scanned >= _words_scanned_limit) ? " (*)" : "",
3479 _refs_reached,
3480 (_refs_reached >= _refs_reached_limit) ? " (*)" : "");
3481 }
3482 #endif // _MARKING_STATS_
3484 // (4) We check whether we should yield. If we have to, then we abort.
3485 if (_cm->should_yield()) {
3486 // We should yield. To do this we abort the task. The caller is
3487 // responsible for yielding.
3488 set_has_aborted();
3489 statsOnly( ++_aborted_yield );
3490 return;
3491 }
3493 // (5) We check whether we've reached our time quota. If we have,
3494 // then we abort.
3495 double elapsed_time_ms = curr_time_ms - _start_time_ms;
3496 if (elapsed_time_ms > _time_target_ms) {
3497 set_has_aborted();
3498 _has_timed_out = true;
3499 statsOnly( ++_aborted_timed_out );
3500 return;
3501 }
3503 // (6) Finally, we check whether there are enough completed STAB
3504 // buffers available for processing. If there are, we abort.
3505 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3506 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) {
3507 if (_cm->verbose_low())
3508 gclog_or_tty->print_cr("[%d] aborting to deal with pending SATB buffers",
3509 _task_id);
3510 // we do need to process SATB buffers, we'll abort and restart
3511 // the marking task to do so
3512 set_has_aborted();
3513 statsOnly( ++_aborted_satb );
3514 return;
3515 }
3516 }
3518 void CMTask::recalculate_limits() {
3519 _real_words_scanned_limit = _words_scanned + words_scanned_period;
3520 _words_scanned_limit = _real_words_scanned_limit;
3522 _real_refs_reached_limit = _refs_reached + refs_reached_period;
3523 _refs_reached_limit = _real_refs_reached_limit;
3524 }
3526 void CMTask::decrease_limits() {
3527 // This is called when we believe that we're going to do an infrequent
3528 // operation which will increase the per byte scanned cost (i.e. move
3529 // entries to/from the global stack). It basically tries to decrease the
3530 // scanning limit so that the clock is called earlier.
3532 if (_cm->verbose_medium())
3533 gclog_or_tty->print_cr("[%d] decreasing limits", _task_id);
3535 _words_scanned_limit = _real_words_scanned_limit -
3536 3 * words_scanned_period / 4;
3537 _refs_reached_limit = _real_refs_reached_limit -
3538 3 * refs_reached_period / 4;
3539 }
3541 void CMTask::move_entries_to_global_stack() {
3542 // local array where we'll store the entries that will be popped
3543 // from the local queue
3544 oop buffer[global_stack_transfer_size];
3546 int n = 0;
3547 oop obj;
3548 while (n < global_stack_transfer_size && _task_queue->pop_local(obj)) {
3549 buffer[n] = obj;
3550 ++n;
3551 }
3553 if (n > 0) {
3554 // we popped at least one entry from the local queue
3556 statsOnly( ++_global_transfers_to; _local_pops += n );
3558 if (!_cm->mark_stack_push(buffer, n)) {
3559 if (_cm->verbose_low())
3560 gclog_or_tty->print_cr("[%d] aborting due to global stack overflow", _task_id);
3561 set_has_aborted();
3562 } else {
3563 // the transfer was successful
3565 if (_cm->verbose_medium())
3566 gclog_or_tty->print_cr("[%d] pushed %d entries to the global stack",
3567 _task_id, n);
3568 statsOnly( int tmp_size = _cm->mark_stack_size();
3569 if (tmp_size > _global_max_size)
3570 _global_max_size = tmp_size;
3571 _global_pushes += n );
3572 }
3573 }
3575 // this operation was quite expensive, so decrease the limits
3576 decrease_limits();
3577 }
3579 void CMTask::get_entries_from_global_stack() {
3580 // local array where we'll store the entries that will be popped
3581 // from the global stack.
3582 oop buffer[global_stack_transfer_size];
3583 int n;
3584 _cm->mark_stack_pop(buffer, global_stack_transfer_size, &n);
3585 assert(n <= global_stack_transfer_size,
3586 "we should not pop more than the given limit");
3587 if (n > 0) {
3588 // yes, we did actually pop at least one entry
3590 statsOnly( ++_global_transfers_from; _global_pops += n );
3591 if (_cm->verbose_medium())
3592 gclog_or_tty->print_cr("[%d] popped %d entries from the global stack",
3593 _task_id, n);
3594 for (int i = 0; i < n; ++i) {
3595 bool success = _task_queue->push(buffer[i]);
3596 // We only call this when the local queue is empty or under a
3597 // given target limit. So, we do not expect this push to fail.
3598 assert(success, "invariant");
3599 }
3601 statsOnly( int tmp_size = _task_queue->size();
3602 if (tmp_size > _local_max_size)
3603 _local_max_size = tmp_size;
3604 _local_pushes += n );
3605 }
3607 // this operation was quite expensive, so decrease the limits
3608 decrease_limits();
3609 }
3611 void CMTask::drain_local_queue(bool partially) {
3612 if (has_aborted())
3613 return;
3615 // Decide what the target size is, depending whether we're going to
3616 // drain it partially (so that other tasks can steal if they run out
3617 // of things to do) or totally (at the very end).
3618 size_t target_size;
3619 if (partially)
3620 target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize);
3621 else
3622 target_size = 0;
3624 if (_task_queue->size() > target_size) {
3625 if (_cm->verbose_high())
3626 gclog_or_tty->print_cr("[%d] draining local queue, target size = %d",
3627 _task_id, target_size);
3629 oop obj;
3630 bool ret = _task_queue->pop_local(obj);
3631 while (ret) {
3632 statsOnly( ++_local_pops );
3634 if (_cm->verbose_high())
3635 gclog_or_tty->print_cr("[%d] popped "PTR_FORMAT, _task_id,
3636 (void*) obj);
3638 assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" );
3639 assert(!_g1h->is_on_free_list(
3640 _g1h->heap_region_containing((HeapWord*) obj)), "invariant");
3642 scan_object(obj);
3644 if (_task_queue->size() <= target_size || has_aborted())
3645 ret = false;
3646 else
3647 ret = _task_queue->pop_local(obj);
3648 }
3650 if (_cm->verbose_high())
3651 gclog_or_tty->print_cr("[%d] drained local queue, size = %d",
3652 _task_id, _task_queue->size());
3653 }
3654 }
3656 void CMTask::drain_global_stack(bool partially) {
3657 if (has_aborted())
3658 return;
3660 // We have a policy to drain the local queue before we attempt to
3661 // drain the global stack.
3662 assert(partially || _task_queue->size() == 0, "invariant");
3664 // Decide what the target size is, depending whether we're going to
3665 // drain it partially (so that other tasks can steal if they run out
3666 // of things to do) or totally (at the very end). Notice that,
3667 // because we move entries from the global stack in chunks or
3668 // because another task might be doing the same, we might in fact
3669 // drop below the target. But, this is not a problem.
3670 size_t target_size;
3671 if (partially)
3672 target_size = _cm->partial_mark_stack_size_target();
3673 else
3674 target_size = 0;
3676 if (_cm->mark_stack_size() > target_size) {
3677 if (_cm->verbose_low())
3678 gclog_or_tty->print_cr("[%d] draining global_stack, target size %d",
3679 _task_id, target_size);
3681 while (!has_aborted() && _cm->mark_stack_size() > target_size) {
3682 get_entries_from_global_stack();
3683 drain_local_queue(partially);
3684 }
3686 if (_cm->verbose_low())
3687 gclog_or_tty->print_cr("[%d] drained global stack, size = %d",
3688 _task_id, _cm->mark_stack_size());
3689 }
3690 }
3692 // SATB Queue has several assumptions on whether to call the par or
3693 // non-par versions of the methods. this is why some of the code is
3694 // replicated. We should really get rid of the single-threaded version
3695 // of the code to simplify things.
3696 void CMTask::drain_satb_buffers() {
3697 if (has_aborted())
3698 return;
3700 // We set this so that the regular clock knows that we're in the
3701 // middle of draining buffers and doesn't set the abort flag when it
3702 // notices that SATB buffers are available for draining. It'd be
3703 // very counter productive if it did that. :-)
3704 _draining_satb_buffers = true;
3706 CMObjectClosure oc(this);
3707 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3708 if (G1CollectedHeap::use_parallel_gc_threads())
3709 satb_mq_set.set_par_closure(_task_id, &oc);
3710 else
3711 satb_mq_set.set_closure(&oc);
3713 // This keeps claiming and applying the closure to completed buffers
3714 // until we run out of buffers or we need to abort.
3715 if (G1CollectedHeap::use_parallel_gc_threads()) {
3716 while (!has_aborted() &&
3717 satb_mq_set.par_apply_closure_to_completed_buffer(_task_id)) {
3718 if (_cm->verbose_medium())
3719 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id);
3720 statsOnly( ++_satb_buffers_processed );
3721 regular_clock_call();
3722 }
3723 } else {
3724 while (!has_aborted() &&
3725 satb_mq_set.apply_closure_to_completed_buffer()) {
3726 if (_cm->verbose_medium())
3727 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id);
3728 statsOnly( ++_satb_buffers_processed );
3729 regular_clock_call();
3730 }
3731 }
3733 if (!concurrent() && !has_aborted()) {
3734 // We should only do this during remark.
3735 if (G1CollectedHeap::use_parallel_gc_threads())
3736 satb_mq_set.par_iterate_closure_all_threads(_task_id);
3737 else
3738 satb_mq_set.iterate_closure_all_threads();
3739 }
3741 _draining_satb_buffers = false;
3743 assert(has_aborted() ||
3744 concurrent() ||
3745 satb_mq_set.completed_buffers_num() == 0, "invariant");
3747 if (G1CollectedHeap::use_parallel_gc_threads())
3748 satb_mq_set.set_par_closure(_task_id, NULL);
3749 else
3750 satb_mq_set.set_closure(NULL);
3752 // again, this was a potentially expensive operation, decrease the
3753 // limits to get the regular clock call early
3754 decrease_limits();
3755 }
3757 void CMTask::drain_region_stack(BitMapClosure* bc) {
3758 if (has_aborted())
3759 return;
3761 assert(_region_finger == NULL,
3762 "it should be NULL when we're not scanning a region");
3764 if (!_cm->region_stack_empty() || !_aborted_region.is_empty()) {
3765 if (_cm->verbose_low())
3766 gclog_or_tty->print_cr("[%d] draining region stack, size = %d",
3767 _task_id, _cm->region_stack_size());
3769 MemRegion mr;
3771 if (!_aborted_region.is_empty()) {
3772 mr = _aborted_region;
3773 _aborted_region = MemRegion();
3775 if (_cm->verbose_low())
3776 gclog_or_tty->print_cr("[%d] scanning aborted region [ " PTR_FORMAT ", " PTR_FORMAT " )",
3777 _task_id, mr.start(), mr.end());
3778 } else {
3779 mr = _cm->region_stack_pop_lock_free();
3780 // it returns MemRegion() if the pop fails
3781 statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
3782 }
3784 while (mr.start() != NULL) {
3785 if (_cm->verbose_medium())
3786 gclog_or_tty->print_cr("[%d] we are scanning region "
3787 "["PTR_FORMAT", "PTR_FORMAT")",
3788 _task_id, mr.start(), mr.end());
3790 assert(mr.end() <= _cm->finger(),
3791 "otherwise the region shouldn't be on the stack");
3792 assert(!mr.is_empty(), "Only non-empty regions live on the region stack");
3793 if (_nextMarkBitMap->iterate(bc, mr)) {
3794 assert(!has_aborted(),
3795 "cannot abort the task without aborting the bitmap iteration");
3797 // We finished iterating over the region without aborting.
3798 regular_clock_call();
3799 if (has_aborted())
3800 mr = MemRegion();
3801 else {
3802 mr = _cm->region_stack_pop_lock_free();
3803 // it returns MemRegion() if the pop fails
3804 statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
3805 }
3806 } else {
3807 assert(has_aborted(), "currently the only way to do so");
3809 // The only way to abort the bitmap iteration is to return
3810 // false from the do_bit() method. However, inside the
3811 // do_bit() method we move the _region_finger to point to the
3812 // object currently being looked at. So, if we bail out, we
3813 // have definitely set _region_finger to something non-null.
3814 assert(_region_finger != NULL, "invariant");
3816 // Make sure that any previously aborted region has been
3817 // cleared.
3818 assert(_aborted_region.is_empty(), "aborted region not cleared");
3820 // The iteration was actually aborted. So now _region_finger
3821 // points to the address of the object we last scanned. If we
3822 // leave it there, when we restart this task, we will rescan
3823 // the object. It is easy to avoid this. We move the finger by
3824 // enough to point to the next possible object header (the
3825 // bitmap knows by how much we need to move it as it knows its
3826 // granularity).
3827 MemRegion newRegion =
3828 MemRegion(_nextMarkBitMap->nextWord(_region_finger), mr.end());
3830 if (!newRegion.is_empty()) {
3831 if (_cm->verbose_low()) {
3832 gclog_or_tty->print_cr("[%d] recording unscanned region"
3833 "[" PTR_FORMAT "," PTR_FORMAT ") in CMTask",
3834 _task_id,
3835 newRegion.start(), newRegion.end());
3836 }
3837 // Now record the part of the region we didn't scan to
3838 // make sure this task scans it later.
3839 _aborted_region = newRegion;
3840 }
3841 // break from while
3842 mr = MemRegion();
3843 }
3844 _region_finger = NULL;
3845 }
3847 if (_cm->verbose_low())
3848 gclog_or_tty->print_cr("[%d] drained region stack, size = %d",
3849 _task_id, _cm->region_stack_size());
3850 }
3851 }
3853 void CMTask::print_stats() {
3854 gclog_or_tty->print_cr("Marking Stats, task = %d, calls = %d",
3855 _task_id, _calls);
3856 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3857 _elapsed_time_ms, _termination_time_ms);
3858 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3859 _step_times_ms.num(), _step_times_ms.avg(),
3860 _step_times_ms.sd());
3861 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
3862 _step_times_ms.maximum(), _step_times_ms.sum());
3864 #if _MARKING_STATS_
3865 gclog_or_tty->print_cr(" Clock Intervals (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3866 _all_clock_intervals_ms.num(), _all_clock_intervals_ms.avg(),
3867 _all_clock_intervals_ms.sd());
3868 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
3869 _all_clock_intervals_ms.maximum(),
3870 _all_clock_intervals_ms.sum());
3871 gclog_or_tty->print_cr(" Clock Causes (cum): scanning = %d, marking = %d",
3872 _clock_due_to_scanning, _clock_due_to_marking);
3873 gclog_or_tty->print_cr(" Objects: scanned = %d, found on the bitmap = %d",
3874 _objs_scanned, _objs_found_on_bitmap);
3875 gclog_or_tty->print_cr(" Local Queue: pushes = %d, pops = %d, max size = %d",
3876 _local_pushes, _local_pops, _local_max_size);
3877 gclog_or_tty->print_cr(" Global Stack: pushes = %d, pops = %d, max size = %d",
3878 _global_pushes, _global_pops, _global_max_size);
3879 gclog_or_tty->print_cr(" transfers to = %d, transfers from = %d",
3880 _global_transfers_to,_global_transfers_from);
3881 gclog_or_tty->print_cr(" Regions: claimed = %d, Region Stack: pops = %d",
3882 _regions_claimed, _region_stack_pops);
3883 gclog_or_tty->print_cr(" SATB buffers: processed = %d", _satb_buffers_processed);
3884 gclog_or_tty->print_cr(" Steals: attempts = %d, successes = %d",
3885 _steal_attempts, _steals);
3886 gclog_or_tty->print_cr(" Aborted: %d, due to", _aborted);
3887 gclog_or_tty->print_cr(" overflow: %d, global abort: %d, yield: %d",
3888 _aborted_overflow, _aborted_cm_aborted, _aborted_yield);
3889 gclog_or_tty->print_cr(" time out: %d, SATB: %d, termination: %d",
3890 _aborted_timed_out, _aborted_satb, _aborted_termination);
3891 #endif // _MARKING_STATS_
3892 }
3894 /*****************************************************************************
3896 The do_marking_step(time_target_ms) method is the building block
3897 of the parallel marking framework. It can be called in parallel
3898 with other invocations of do_marking_step() on different tasks
3899 (but only one per task, obviously) and concurrently with the
3900 mutator threads, or during remark, hence it eliminates the need
3901 for two versions of the code. When called during remark, it will
3902 pick up from where the task left off during the concurrent marking
3903 phase. Interestingly, tasks are also claimable during evacuation
3904 pauses too, since do_marking_step() ensures that it aborts before
3905 it needs to yield.
3907 The data structures that is uses to do marking work are the
3908 following:
3910 (1) Marking Bitmap. If there are gray objects that appear only
3911 on the bitmap (this happens either when dealing with an overflow
3912 or when the initial marking phase has simply marked the roots
3913 and didn't push them on the stack), then tasks claim heap
3914 regions whose bitmap they then scan to find gray objects. A
3915 global finger indicates where the end of the last claimed region
3916 is. A local finger indicates how far into the region a task has
3917 scanned. The two fingers are used to determine how to gray an
3918 object (i.e. whether simply marking it is OK, as it will be
3919 visited by a task in the future, or whether it needs to be also
3920 pushed on a stack).
3922 (2) Local Queue. The local queue of the task which is accessed
3923 reasonably efficiently by the task. Other tasks can steal from
3924 it when they run out of work. Throughout the marking phase, a
3925 task attempts to keep its local queue short but not totally
3926 empty, so that entries are available for stealing by other
3927 tasks. Only when there is no more work, a task will totally
3928 drain its local queue.
3930 (3) Global Mark Stack. This handles local queue overflow. During
3931 marking only sets of entries are moved between it and the local
3932 queues, as access to it requires a mutex and more fine-grain
3933 interaction with it which might cause contention. If it
3934 overflows, then the marking phase should restart and iterate
3935 over the bitmap to identify gray objects. Throughout the marking
3936 phase, tasks attempt to keep the global mark stack at a small
3937 length but not totally empty, so that entries are available for
3938 popping by other tasks. Only when there is no more work, tasks
3939 will totally drain the global mark stack.
3941 (4) Global Region Stack. Entries on it correspond to areas of
3942 the bitmap that need to be scanned since they contain gray
3943 objects. Pushes on the region stack only happen during
3944 evacuation pauses and typically correspond to areas covered by
3945 GC LABS. If it overflows, then the marking phase should restart
3946 and iterate over the bitmap to identify gray objects. Tasks will
3947 try to totally drain the region stack as soon as possible.
3949 (5) SATB Buffer Queue. This is where completed SATB buffers are
3950 made available. Buffers are regularly removed from this queue
3951 and scanned for roots, so that the queue doesn't get too
3952 long. During remark, all completed buffers are processed, as
3953 well as the filled in parts of any uncompleted buffers.
3955 The do_marking_step() method tries to abort when the time target
3956 has been reached. There are a few other cases when the
3957 do_marking_step() method also aborts:
3959 (1) When the marking phase has been aborted (after a Full GC).
3961 (2) When a global overflow (either on the global stack or the
3962 region stack) has been triggered. Before the task aborts, it
3963 will actually sync up with the other tasks to ensure that all
3964 the marking data structures (local queues, stacks, fingers etc.)
3965 are re-initialised so that when do_marking_step() completes,
3966 the marking phase can immediately restart.
3968 (3) When enough completed SATB buffers are available. The
3969 do_marking_step() method only tries to drain SATB buffers right
3970 at the beginning. So, if enough buffers are available, the
3971 marking step aborts and the SATB buffers are processed at
3972 the beginning of the next invocation.
3974 (4) To yield. when we have to yield then we abort and yield
3975 right at the end of do_marking_step(). This saves us from a lot
3976 of hassle as, by yielding we might allow a Full GC. If this
3977 happens then objects will be compacted underneath our feet, the
3978 heap might shrink, etc. We save checking for this by just
3979 aborting and doing the yield right at the end.
3981 From the above it follows that the do_marking_step() method should
3982 be called in a loop (or, otherwise, regularly) until it completes.
3984 If a marking step completes without its has_aborted() flag being
3985 true, it means it has completed the current marking phase (and
3986 also all other marking tasks have done so and have all synced up).
3988 A method called regular_clock_call() is invoked "regularly" (in
3989 sub ms intervals) throughout marking. It is this clock method that
3990 checks all the abort conditions which were mentioned above and
3991 decides when the task should abort. A work-based scheme is used to
3992 trigger this clock method: when the number of object words the
3993 marking phase has scanned or the number of references the marking
3994 phase has visited reach a given limit. Additional invocations to
3995 the method clock have been planted in a few other strategic places
3996 too. The initial reason for the clock method was to avoid calling
3997 vtime too regularly, as it is quite expensive. So, once it was in
3998 place, it was natural to piggy-back all the other conditions on it
3999 too and not constantly check them throughout the code.
4001 *****************************************************************************/
4003 void CMTask::do_marking_step(double time_target_ms,
4004 bool do_stealing,
4005 bool do_termination) {
4006 assert(time_target_ms >= 1.0, "minimum granularity is 1ms");
4007 assert(concurrent() == _cm->concurrent(), "they should be the same");
4009 assert(concurrent() || _cm->region_stack_empty(),
4010 "the region stack should have been cleared before remark");
4011 assert(concurrent() || !_cm->has_aborted_regions(),
4012 "aborted regions should have been cleared before remark");
4013 assert(_region_finger == NULL,
4014 "this should be non-null only when a region is being scanned");
4016 G1CollectorPolicy* g1_policy = _g1h->g1_policy();
4017 assert(_task_queues != NULL, "invariant");
4018 assert(_task_queue != NULL, "invariant");
4019 assert(_task_queues->queue(_task_id) == _task_queue, "invariant");
4021 assert(!_claimed,
4022 "only one thread should claim this task at any one time");
4024 // OK, this doesn't safeguard again all possible scenarios, as it is
4025 // possible for two threads to set the _claimed flag at the same
4026 // time. But it is only for debugging purposes anyway and it will
4027 // catch most problems.
4028 _claimed = true;
4030 _start_time_ms = os::elapsedVTime() * 1000.0;
4031 statsOnly( _interval_start_time_ms = _start_time_ms );
4033 double diff_prediction_ms =
4034 g1_policy->get_new_prediction(&_marking_step_diffs_ms);
4035 _time_target_ms = time_target_ms - diff_prediction_ms;
4037 // set up the variables that are used in the work-based scheme to
4038 // call the regular clock method
4039 _words_scanned = 0;
4040 _refs_reached = 0;
4041 recalculate_limits();
4043 // clear all flags
4044 clear_has_aborted();
4045 _has_timed_out = false;
4046 _draining_satb_buffers = false;
4048 ++_calls;
4050 if (_cm->verbose_low())
4051 gclog_or_tty->print_cr("[%d] >>>>>>>>>> START, call = %d, "
4052 "target = %1.2lfms >>>>>>>>>>",
4053 _task_id, _calls, _time_target_ms);
4055 // Set up the bitmap and oop closures. Anything that uses them is
4056 // eventually called from this method, so it is OK to allocate these
4057 // statically.
4058 CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap);
4059 CMOopClosure oop_closure(_g1h, _cm, this);
4060 set_oop_closure(&oop_closure);
4062 if (_cm->has_overflown()) {
4063 // This can happen if the region stack or the mark stack overflows
4064 // during a GC pause and this task, after a yield point,
4065 // restarts. We have to abort as we need to get into the overflow
4066 // protocol which happens right at the end of this task.
4067 set_has_aborted();
4068 }
4070 // First drain any available SATB buffers. After this, we will not
4071 // look at SATB buffers before the next invocation of this method.
4072 // If enough completed SATB buffers are queued up, the regular clock
4073 // will abort this task so that it restarts.
4074 drain_satb_buffers();
4075 // ...then partially drain the local queue and the global stack
4076 drain_local_queue(true);
4077 drain_global_stack(true);
4079 // Then totally drain the region stack. We will not look at
4080 // it again before the next invocation of this method. Entries on
4081 // the region stack are only added during evacuation pauses, for
4082 // which we have to yield. When we do, we abort the task anyway so
4083 // it will look at the region stack again when it restarts.
4084 bitmap_closure.set_scanning_heap_region(false);
4085 drain_region_stack(&bitmap_closure);
4086 // ...then partially drain the local queue and the global stack
4087 drain_local_queue(true);
4088 drain_global_stack(true);
4090 do {
4091 if (!has_aborted() && _curr_region != NULL) {
4092 // This means that we're already holding on to a region.
4093 assert(_finger != NULL, "if region is not NULL, then the finger "
4094 "should not be NULL either");
4096 // We might have restarted this task after an evacuation pause
4097 // which might have evacuated the region we're holding on to
4098 // underneath our feet. Let's read its limit again to make sure
4099 // that we do not iterate over a region of the heap that
4100 // contains garbage (update_region_limit() will also move
4101 // _finger to the start of the region if it is found empty).
4102 update_region_limit();
4103 // We will start from _finger not from the start of the region,
4104 // as we might be restarting this task after aborting half-way
4105 // through scanning this region. In this case, _finger points to
4106 // the address where we last found a marked object. If this is a
4107 // fresh region, _finger points to start().
4108 MemRegion mr = MemRegion(_finger, _region_limit);
4110 if (_cm->verbose_low())
4111 gclog_or_tty->print_cr("[%d] we're scanning part "
4112 "["PTR_FORMAT", "PTR_FORMAT") "
4113 "of region "PTR_FORMAT,
4114 _task_id, _finger, _region_limit, _curr_region);
4116 // Let's iterate over the bitmap of the part of the
4117 // region that is left.
4118 bitmap_closure.set_scanning_heap_region(true);
4119 if (mr.is_empty() ||
4120 _nextMarkBitMap->iterate(&bitmap_closure, mr)) {
4121 // We successfully completed iterating over the region. Now,
4122 // let's give up the region.
4123 giveup_current_region();
4124 regular_clock_call();
4125 } else {
4126 assert(has_aborted(), "currently the only way to do so");
4127 // The only way to abort the bitmap iteration is to return
4128 // false from the do_bit() method. However, inside the
4129 // do_bit() method we move the _finger to point to the
4130 // object currently being looked at. So, if we bail out, we
4131 // have definitely set _finger to something non-null.
4132 assert(_finger != NULL, "invariant");
4134 // Region iteration was actually aborted. So now _finger
4135 // points to the address of the object we last scanned. If we
4136 // leave it there, when we restart this task, we will rescan
4137 // the object. It is easy to avoid this. We move the finger by
4138 // enough to point to the next possible object header (the
4139 // bitmap knows by how much we need to move it as it knows its
4140 // granularity).
4141 assert(_finger < _region_limit, "invariant");
4142 HeapWord* new_finger = _nextMarkBitMap->nextWord(_finger);
4143 // Check if bitmap iteration was aborted while scanning the last object
4144 if (new_finger >= _region_limit) {
4145 giveup_current_region();
4146 } else {
4147 move_finger_to(new_finger);
4148 }
4149 }
4150 }
4151 // At this point we have either completed iterating over the
4152 // region we were holding on to, or we have aborted.
4154 // We then partially drain the local queue and the global stack.
4155 // (Do we really need this?)
4156 drain_local_queue(true);
4157 drain_global_stack(true);
4159 // Read the note on the claim_region() method on why it might
4160 // return NULL with potentially more regions available for
4161 // claiming and why we have to check out_of_regions() to determine
4162 // whether we're done or not.
4163 while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) {
4164 // We are going to try to claim a new region. We should have
4165 // given up on the previous one.
4166 // Separated the asserts so that we know which one fires.
4167 assert(_curr_region == NULL, "invariant");
4168 assert(_finger == NULL, "invariant");
4169 assert(_region_limit == NULL, "invariant");
4170 if (_cm->verbose_low())
4171 gclog_or_tty->print_cr("[%d] trying to claim a new region", _task_id);
4172 HeapRegion* claimed_region = _cm->claim_region(_task_id);
4173 if (claimed_region != NULL) {
4174 // Yes, we managed to claim one
4175 statsOnly( ++_regions_claimed );
4177 if (_cm->verbose_low())
4178 gclog_or_tty->print_cr("[%d] we successfully claimed "
4179 "region "PTR_FORMAT,
4180 _task_id, claimed_region);
4182 setup_for_region(claimed_region);
4183 assert(_curr_region == claimed_region, "invariant");
4184 }
4185 // It is important to call the regular clock here. It might take
4186 // a while to claim a region if, for example, we hit a large
4187 // block of empty regions. So we need to call the regular clock
4188 // method once round the loop to make sure it's called
4189 // frequently enough.
4190 regular_clock_call();
4191 }
4193 if (!has_aborted() && _curr_region == NULL) {
4194 assert(_cm->out_of_regions(),
4195 "at this point we should be out of regions");
4196 }
4197 } while ( _curr_region != NULL && !has_aborted());
4199 if (!has_aborted()) {
4200 // We cannot check whether the global stack is empty, since other
4201 // tasks might be pushing objects to it concurrently. We also cannot
4202 // check if the region stack is empty because if a thread is aborting
4203 // it can push a partially done region back.
4204 assert(_cm->out_of_regions(),
4205 "at this point we should be out of regions");
4207 if (_cm->verbose_low())
4208 gclog_or_tty->print_cr("[%d] all regions claimed", _task_id);
4210 // Try to reduce the number of available SATB buffers so that
4211 // remark has less work to do.
4212 drain_satb_buffers();
4213 }
4215 // Since we've done everything else, we can now totally drain the
4216 // local queue and global stack.
4217 drain_local_queue(false);
4218 drain_global_stack(false);
4220 // Attempt at work stealing from other task's queues.
4221 if (do_stealing && !has_aborted()) {
4222 // We have not aborted. This means that we have finished all that
4223 // we could. Let's try to do some stealing...
4225 // We cannot check whether the global stack is empty, since other
4226 // tasks might be pushing objects to it concurrently. We also cannot
4227 // check if the region stack is empty because if a thread is aborting
4228 // it can push a partially done region back.
4229 assert(_cm->out_of_regions() && _task_queue->size() == 0,
4230 "only way to reach here");
4232 if (_cm->verbose_low())
4233 gclog_or_tty->print_cr("[%d] starting to steal", _task_id);
4235 while (!has_aborted()) {
4236 oop obj;
4237 statsOnly( ++_steal_attempts );
4239 if (_cm->try_stealing(_task_id, &_hash_seed, obj)) {
4240 if (_cm->verbose_medium())
4241 gclog_or_tty->print_cr("[%d] stolen "PTR_FORMAT" successfully",
4242 _task_id, (void*) obj);
4244 statsOnly( ++_steals );
4246 assert(_nextMarkBitMap->isMarked((HeapWord*) obj),
4247 "any stolen object should be marked");
4248 scan_object(obj);
4250 // And since we're towards the end, let's totally drain the
4251 // local queue and global stack.
4252 drain_local_queue(false);
4253 drain_global_stack(false);
4254 } else {
4255 break;
4256 }
4257 }
4258 }
4260 // We still haven't aborted. Now, let's try to get into the
4261 // termination protocol.
4262 if (do_termination && !has_aborted()) {
4263 // We cannot check whether the global stack is empty, since other
4264 // tasks might be concurrently pushing objects on it. We also cannot
4265 // check if the region stack is empty because if a thread is aborting
4266 // it can push a partially done region back.
4267 // Separated the asserts so that we know which one fires.
4268 assert(_cm->out_of_regions(), "only way to reach here");
4269 assert(_task_queue->size() == 0, "only way to reach here");
4271 if (_cm->verbose_low())
4272 gclog_or_tty->print_cr("[%d] starting termination protocol", _task_id);
4274 _termination_start_time_ms = os::elapsedVTime() * 1000.0;
4275 // The CMTask class also extends the TerminatorTerminator class,
4276 // hence its should_exit_termination() method will also decide
4277 // whether to exit the termination protocol or not.
4278 bool finished = _cm->terminator()->offer_termination(this);
4279 double termination_end_time_ms = os::elapsedVTime() * 1000.0;
4280 _termination_time_ms +=
4281 termination_end_time_ms - _termination_start_time_ms;
4283 if (finished) {
4284 // We're all done.
4286 if (_task_id == 0) {
4287 // let's allow task 0 to do this
4288 if (concurrent()) {
4289 assert(_cm->concurrent_marking_in_progress(), "invariant");
4290 // we need to set this to false before the next
4291 // safepoint. This way we ensure that the marking phase
4292 // doesn't observe any more heap expansions.
4293 _cm->clear_concurrent_marking_in_progress();
4294 }
4295 }
4297 // We can now guarantee that the global stack is empty, since
4298 // all other tasks have finished. We separated the guarantees so
4299 // that, if a condition is false, we can immediately find out
4300 // which one.
4301 guarantee(_cm->out_of_regions(), "only way to reach here");
4302 guarantee(_aborted_region.is_empty(), "only way to reach here");
4303 guarantee(_cm->region_stack_empty(), "only way to reach here");
4304 guarantee(_cm->mark_stack_empty(), "only way to reach here");
4305 guarantee(_task_queue->size() == 0, "only way to reach here");
4306 guarantee(!_cm->has_overflown(), "only way to reach here");
4307 guarantee(!_cm->mark_stack_overflow(), "only way to reach here");
4308 guarantee(!_cm->region_stack_overflow(), "only way to reach here");
4310 if (_cm->verbose_low())
4311 gclog_or_tty->print_cr("[%d] all tasks terminated", _task_id);
4312 } else {
4313 // Apparently there's more work to do. Let's abort this task. It
4314 // will restart it and we can hopefully find more things to do.
4316 if (_cm->verbose_low())
4317 gclog_or_tty->print_cr("[%d] apparently there is more work to do", _task_id);
4319 set_has_aborted();
4320 statsOnly( ++_aborted_termination );
4321 }
4322 }
4324 // Mainly for debugging purposes to make sure that a pointer to the
4325 // closure which was statically allocated in this frame doesn't
4326 // escape it by accident.
4327 set_oop_closure(NULL);
4328 double end_time_ms = os::elapsedVTime() * 1000.0;
4329 double elapsed_time_ms = end_time_ms - _start_time_ms;
4330 // Update the step history.
4331 _step_times_ms.add(elapsed_time_ms);
4333 if (has_aborted()) {
4334 // The task was aborted for some reason.
4336 statsOnly( ++_aborted );
4338 if (_has_timed_out) {
4339 double diff_ms = elapsed_time_ms - _time_target_ms;
4340 // Keep statistics of how well we did with respect to hitting
4341 // our target only if we actually timed out (if we aborted for
4342 // other reasons, then the results might get skewed).
4343 _marking_step_diffs_ms.add(diff_ms);
4344 }
4346 if (_cm->has_overflown()) {
4347 // This is the interesting one. We aborted because a global
4348 // overflow was raised. This means we have to restart the
4349 // marking phase and start iterating over regions. However, in
4350 // order to do this we have to make sure that all tasks stop
4351 // what they are doing and re-initialise in a safe manner. We
4352 // will achieve this with the use of two barrier sync points.
4354 if (_cm->verbose_low())
4355 gclog_or_tty->print_cr("[%d] detected overflow", _task_id);
4357 _cm->enter_first_sync_barrier(_task_id);
4358 // When we exit this sync barrier we know that all tasks have
4359 // stopped doing marking work. So, it's now safe to
4360 // re-initialise our data structures. At the end of this method,
4361 // task 0 will clear the global data structures.
4363 statsOnly( ++_aborted_overflow );
4365 // We clear the local state of this task...
4366 clear_region_fields();
4368 // ...and enter the second barrier.
4369 _cm->enter_second_sync_barrier(_task_id);
4370 // At this point everything has bee re-initialised and we're
4371 // ready to restart.
4372 }
4374 if (_cm->verbose_low()) {
4375 gclog_or_tty->print_cr("[%d] <<<<<<<<<< ABORTING, target = %1.2lfms, "
4376 "elapsed = %1.2lfms <<<<<<<<<<",
4377 _task_id, _time_target_ms, elapsed_time_ms);
4378 if (_cm->has_aborted())
4379 gclog_or_tty->print_cr("[%d] ========== MARKING ABORTED ==========",
4380 _task_id);
4381 }
4382 } else {
4383 if (_cm->verbose_low())
4384 gclog_or_tty->print_cr("[%d] <<<<<<<<<< FINISHED, target = %1.2lfms, "
4385 "elapsed = %1.2lfms <<<<<<<<<<",
4386 _task_id, _time_target_ms, elapsed_time_ms);
4387 }
4389 _claimed = false;
4390 }
4392 CMTask::CMTask(int task_id,
4393 ConcurrentMark* cm,
4394 CMTaskQueue* task_queue,
4395 CMTaskQueueSet* task_queues)
4396 : _g1h(G1CollectedHeap::heap()),
4397 _task_id(task_id), _cm(cm),
4398 _claimed(false),
4399 _nextMarkBitMap(NULL), _hash_seed(17),
4400 _task_queue(task_queue),
4401 _task_queues(task_queues),
4402 _oop_closure(NULL),
4403 _aborted_region(MemRegion()) {
4404 guarantee(task_queue != NULL, "invariant");
4405 guarantee(task_queues != NULL, "invariant");
4407 statsOnly( _clock_due_to_scanning = 0;
4408 _clock_due_to_marking = 0 );
4410 _marking_step_diffs_ms.add(0.5);
4411 }