ysr@777: /* johnc@4386: * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. ysr@777: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ysr@777: * ysr@777: * This code is free software; you can redistribute it and/or modify it ysr@777: * under the terms of the GNU General Public License version 2 only, as ysr@777: * published by the Free Software Foundation. ysr@777: * ysr@777: * This code is distributed in the hope that it will be useful, but WITHOUT ysr@777: * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ysr@777: * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ysr@777: * version 2 for more details (a copy is included in the LICENSE file that ysr@777: * accompanied this code). ysr@777: * ysr@777: * You should have received a copy of the GNU General Public License version ysr@777: * 2 along with this work; if not, write to the Free Software Foundation, ysr@777: * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ysr@777: * trims@1907: * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA trims@1907: * or visit www.oracle.com if you need additional information or have any trims@1907: * questions. ysr@777: * ysr@777: */ ysr@777: stefank@2314: #include "precompiled.hpp" stefank@2314: #include "classfile/symbolTable.hpp" tonyp@2968: #include "gc_implementation/g1/concurrentMark.inline.hpp" stefank@2314: #include "gc_implementation/g1/concurrentMarkThread.inline.hpp" stefank@2314: #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" stefank@2314: #include "gc_implementation/g1/g1CollectorPolicy.hpp" tonyp@3114: #include "gc_implementation/g1/g1ErgoVerbose.hpp" brutisso@3710: #include "gc_implementation/g1/g1Log.hpp" tonyp@2968: #include "gc_implementation/g1/g1OopClosures.inline.hpp" stefank@2314: #include "gc_implementation/g1/g1RemSet.hpp" tonyp@3416: #include "gc_implementation/g1/heapRegion.inline.hpp" stefank@2314: #include "gc_implementation/g1/heapRegionRemSet.hpp" stefank@2314: #include "gc_implementation/g1/heapRegionSeq.inline.hpp" kamg@2445: #include "gc_implementation/shared/vmGCOperations.hpp" sla@5237: #include "gc_implementation/shared/gcTimer.hpp" sla@5237: #include "gc_implementation/shared/gcTrace.hpp" sla@5237: #include "gc_implementation/shared/gcTraceTime.hpp" stefank@2314: #include "memory/genOopClosures.inline.hpp" stefank@2314: #include "memory/referencePolicy.hpp" stefank@2314: #include "memory/resourceArea.hpp" stefank@2314: #include "oops/oop.inline.hpp" stefank@2314: #include "runtime/handles.inline.hpp" stefank@2314: #include "runtime/java.hpp" zgu@3900: #include "services/memTracker.hpp" ysr@777: brutisso@3455: // Concurrent marking bit map wrapper ysr@777: johnc@4333: CMBitMapRO::CMBitMapRO(int shifter) : johnc@4333: _bm(), ysr@777: _shifter(shifter) { johnc@4333: _bmStartWord = 0; johnc@4333: _bmWordSize = 0; ysr@777: } ysr@777: ysr@777: HeapWord* CMBitMapRO::getNextMarkedWordAddress(HeapWord* addr, ysr@777: HeapWord* limit) const { ysr@777: // First we must round addr *up* to a possible object boundary. ysr@777: addr = (HeapWord*)align_size_up((intptr_t)addr, ysr@777: HeapWordSize << _shifter); ysr@777: size_t addrOffset = heapWordToOffset(addr); tonyp@2973: if (limit == NULL) { tonyp@2973: limit = _bmStartWord + _bmWordSize; tonyp@2973: } ysr@777: size_t limitOffset = heapWordToOffset(limit); ysr@777: size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset); ysr@777: HeapWord* nextAddr = offsetToHeapWord(nextOffset); ysr@777: assert(nextAddr >= addr, "get_next_one postcondition"); ysr@777: assert(nextAddr == limit || isMarked(nextAddr), ysr@777: "get_next_one postcondition"); ysr@777: return nextAddr; ysr@777: } ysr@777: ysr@777: HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(HeapWord* addr, ysr@777: HeapWord* limit) const { ysr@777: size_t addrOffset = heapWordToOffset(addr); tonyp@2973: if (limit == NULL) { tonyp@2973: limit = _bmStartWord + _bmWordSize; tonyp@2973: } ysr@777: size_t limitOffset = heapWordToOffset(limit); ysr@777: size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset); ysr@777: HeapWord* nextAddr = offsetToHeapWord(nextOffset); ysr@777: assert(nextAddr >= addr, "get_next_one postcondition"); ysr@777: assert(nextAddr == limit || !isMarked(nextAddr), ysr@777: "get_next_one postcondition"); ysr@777: return nextAddr; ysr@777: } ysr@777: ysr@777: int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const { ysr@777: assert((diff & ((1 << _shifter) - 1)) == 0, "argument check"); ysr@777: return (int) (diff >> _shifter); ysr@777: } ysr@777: ysr@777: #ifndef PRODUCT johnc@4333: bool CMBitMapRO::covers(ReservedSpace heap_rs) const { ysr@777: // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); brutisso@4061: assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize, ysr@777: "size inconsistency"); johnc@4333: return _bmStartWord == (HeapWord*)(heap_rs.base()) && johnc@4333: _bmWordSize == heap_rs.size()>>LogHeapWordSize; ysr@777: } ysr@777: #endif ysr@777: stefank@4904: void CMBitMapRO::print_on_error(outputStream* st, const char* prefix) const { stefank@4904: _bm.print_on_error(st, prefix); stefank@4904: } stefank@4904: johnc@4333: bool CMBitMap::allocate(ReservedSpace heap_rs) { johnc@4333: _bmStartWord = (HeapWord*)(heap_rs.base()); johnc@4333: _bmWordSize = heap_rs.size()/HeapWordSize; // heap_rs.size() is in bytes johnc@4333: ReservedSpace brs(ReservedSpace::allocation_align_size_up( johnc@4333: (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1)); johnc@4333: if (!brs.is_reserved()) { johnc@4333: warning("ConcurrentMark marking bit map allocation failure"); johnc@4333: return false; johnc@4333: } johnc@4333: MemTracker::record_virtual_memory_type((address)brs.base(), mtGC); johnc@4333: // For now we'll just commit all of the bit map up front. johnc@4333: // Later on we'll try to be more parsimonious with swap. johnc@4333: if (!_virtual_space.initialize(brs, brs.size())) { johnc@4333: warning("ConcurrentMark marking bit map backing store failure"); johnc@4333: return false; johnc@4333: } johnc@4333: assert(_virtual_space.committed_size() == brs.size(), johnc@4333: "didn't reserve backing store for all of concurrent marking bit map?"); johnc@4333: _bm.set_map((uintptr_t*)_virtual_space.low()); johnc@4333: assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >= johnc@4333: _bmWordSize, "inconsistency in bit map sizing"); johnc@4333: _bm.set_size(_bmWordSize >> _shifter); johnc@4333: return true; johnc@4333: } johnc@4333: ysr@777: void CMBitMap::clearAll() { ysr@777: _bm.clear(); ysr@777: return; ysr@777: } ysr@777: ysr@777: void CMBitMap::markRange(MemRegion mr) { ysr@777: mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); ysr@777: assert(!mr.is_empty(), "unexpected empty region"); ysr@777: assert((offsetToHeapWord(heapWordToOffset(mr.end())) == ysr@777: ((HeapWord *) mr.end())), ysr@777: "markRange memory region end is not card aligned"); ysr@777: // convert address range into offset range ysr@777: _bm.at_put_range(heapWordToOffset(mr.start()), ysr@777: heapWordToOffset(mr.end()), true); ysr@777: } ysr@777: ysr@777: void CMBitMap::clearRange(MemRegion mr) { ysr@777: mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); ysr@777: assert(!mr.is_empty(), "unexpected empty region"); ysr@777: // convert address range into offset range ysr@777: _bm.at_put_range(heapWordToOffset(mr.start()), ysr@777: heapWordToOffset(mr.end()), false); ysr@777: } ysr@777: ysr@777: MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr, ysr@777: HeapWord* end_addr) { ysr@777: HeapWord* start = getNextMarkedWordAddress(addr); ysr@777: start = MIN2(start, end_addr); ysr@777: HeapWord* end = getNextUnmarkedWordAddress(start); ysr@777: end = MIN2(end, end_addr); ysr@777: assert(start <= end, "Consistency check"); ysr@777: MemRegion mr(start, end); ysr@777: if (!mr.is_empty()) { ysr@777: clearRange(mr); ysr@777: } ysr@777: return mr; ysr@777: } ysr@777: ysr@777: CMMarkStack::CMMarkStack(ConcurrentMark* cm) : ysr@777: _base(NULL), _cm(cm) ysr@777: #ifdef ASSERT ysr@777: , _drain_in_progress(false) ysr@777: , _drain_in_progress_yields(false) ysr@777: #endif ysr@777: {} ysr@777: johnc@4333: bool CMMarkStack::allocate(size_t capacity) { johnc@4333: // allocate a stack of the requisite depth johnc@4333: ReservedSpace rs(ReservedSpace::allocation_align_size_up(capacity * sizeof(oop))); johnc@4333: if (!rs.is_reserved()) { johnc@4333: warning("ConcurrentMark MarkStack allocation failure"); johnc@4333: return false; tonyp@2973: } johnc@4333: MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); johnc@4333: if (!_virtual_space.initialize(rs, rs.size())) { johnc@4333: warning("ConcurrentMark MarkStack backing store failure"); johnc@4333: // Release the virtual memory reserved for the marking stack johnc@4333: rs.release(); johnc@4333: return false; johnc@4333: } johnc@4333: assert(_virtual_space.committed_size() == rs.size(), johnc@4333: "Didn't reserve backing store for all of ConcurrentMark stack?"); johnc@4333: _base = (oop*) _virtual_space.low(); johnc@4333: setEmpty(); johnc@4333: _capacity = (jint) capacity; tonyp@3416: _saved_index = -1; johnc@4386: _should_expand = false; ysr@777: NOT_PRODUCT(_max_depth = 0); johnc@4333: return true; johnc@4333: } johnc@4333: johnc@4333: void CMMarkStack::expand() { johnc@4333: // Called, during remark, if we've overflown the marking stack during marking. johnc@4333: assert(isEmpty(), "stack should been emptied while handling overflow"); johnc@4333: assert(_capacity <= (jint) MarkStackSizeMax, "stack bigger than permitted"); johnc@4333: // Clear expansion flag johnc@4333: _should_expand = false; johnc@4333: if (_capacity == (jint) MarkStackSizeMax) { johnc@4333: if (PrintGCDetails && Verbose) { johnc@4333: gclog_or_tty->print_cr(" (benign) Can't expand marking stack capacity, at max size limit"); johnc@4333: } johnc@4333: return; johnc@4333: } johnc@4333: // Double capacity if possible johnc@4333: jint new_capacity = MIN2(_capacity*2, (jint) MarkStackSizeMax); johnc@4333: // Do not give up existing stack until we have managed to johnc@4333: // get the double capacity that we desired. johnc@4333: ReservedSpace rs(ReservedSpace::allocation_align_size_up(new_capacity * johnc@4333: sizeof(oop))); johnc@4333: if (rs.is_reserved()) { johnc@4333: // Release the backing store associated with old stack johnc@4333: _virtual_space.release(); johnc@4333: // Reinitialize virtual space for new stack johnc@4333: if (!_virtual_space.initialize(rs, rs.size())) { johnc@4333: fatal("Not enough swap for expanded marking stack capacity"); johnc@4333: } johnc@4333: _base = (oop*)(_virtual_space.low()); johnc@4333: _index = 0; johnc@4333: _capacity = new_capacity; johnc@4333: } else { johnc@4333: if (PrintGCDetails && Verbose) { johnc@4333: // Failed to double capacity, continue; johnc@4333: gclog_or_tty->print(" (benign) Failed to expand marking stack capacity from " johnc@4333: SIZE_FORMAT"K to " SIZE_FORMAT"K", johnc@4333: _capacity / K, new_capacity / K); johnc@4333: } johnc@4333: } johnc@4333: } johnc@4333: johnc@4333: void CMMarkStack::set_should_expand() { johnc@4333: // If we're resetting the marking state because of an johnc@4333: // marking stack overflow, record that we should, if johnc@4333: // possible, expand the stack. johnc@4333: _should_expand = _cm->has_overflown(); ysr@777: } ysr@777: ysr@777: CMMarkStack::~CMMarkStack() { tonyp@2973: if (_base != NULL) { johnc@4333: _base = NULL; johnc@4333: _virtual_space.release(); tonyp@2973: } ysr@777: } ysr@777: ysr@777: void CMMarkStack::par_push(oop ptr) { ysr@777: while (true) { ysr@777: if (isFull()) { ysr@777: _overflow = true; ysr@777: return; ysr@777: } ysr@777: // Otherwise... ysr@777: jint index = _index; ysr@777: jint next_index = index+1; ysr@777: jint res = Atomic::cmpxchg(next_index, &_index, index); ysr@777: if (res == index) { ysr@777: _base[index] = ptr; ysr@777: // Note that we don't maintain this atomically. We could, but it ysr@777: // doesn't seem necessary. ysr@777: NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index)); ysr@777: return; ysr@777: } ysr@777: // Otherwise, we need to try again. ysr@777: } ysr@777: } ysr@777: ysr@777: void CMMarkStack::par_adjoin_arr(oop* ptr_arr, int n) { ysr@777: while (true) { ysr@777: if (isFull()) { ysr@777: _overflow = true; ysr@777: return; ysr@777: } ysr@777: // Otherwise... ysr@777: jint index = _index; ysr@777: jint next_index = index + n; ysr@777: if (next_index > _capacity) { ysr@777: _overflow = true; ysr@777: return; ysr@777: } ysr@777: jint res = Atomic::cmpxchg(next_index, &_index, index); ysr@777: if (res == index) { ysr@777: for (int i = 0; i < n; i++) { johnc@4333: int ind = index + i; ysr@777: assert(ind < _capacity, "By overflow test above."); ysr@777: _base[ind] = ptr_arr[i]; ysr@777: } ysr@777: NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index)); ysr@777: return; ysr@777: } ysr@777: // Otherwise, we need to try again. ysr@777: } ysr@777: } ysr@777: ysr@777: void CMMarkStack::par_push_arr(oop* ptr_arr, int n) { ysr@777: MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); ysr@777: jint start = _index; ysr@777: jint next_index = start + n; ysr@777: if (next_index > _capacity) { ysr@777: _overflow = true; ysr@777: return; ysr@777: } ysr@777: // Otherwise. ysr@777: _index = next_index; ysr@777: for (int i = 0; i < n; i++) { ysr@777: int ind = start + i; tonyp@1458: assert(ind < _capacity, "By overflow test above."); ysr@777: _base[ind] = ptr_arr[i]; ysr@777: } johnc@4333: NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index)); ysr@777: } ysr@777: ysr@777: bool CMMarkStack::par_pop_arr(oop* ptr_arr, int max, int* n) { ysr@777: MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); ysr@777: jint index = _index; ysr@777: if (index == 0) { ysr@777: *n = 0; ysr@777: return false; ysr@777: } else { ysr@777: int k = MIN2(max, index); johnc@4333: jint new_ind = index - k; ysr@777: for (int j = 0; j < k; j++) { ysr@777: ptr_arr[j] = _base[new_ind + j]; ysr@777: } ysr@777: _index = new_ind; ysr@777: *n = k; ysr@777: return true; ysr@777: } ysr@777: } ysr@777: ysr@777: template ysr@777: bool CMMarkStack::drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after) { ysr@777: assert(!_drain_in_progress || !_drain_in_progress_yields || yield_after ysr@777: || SafepointSynchronize::is_at_safepoint(), ysr@777: "Drain recursion must be yield-safe."); ysr@777: bool res = true; ysr@777: debug_only(_drain_in_progress = true); ysr@777: debug_only(_drain_in_progress_yields = yield_after); ysr@777: while (!isEmpty()) { ysr@777: oop newOop = pop(); ysr@777: assert(G1CollectedHeap::heap()->is_in_reserved(newOop), "Bad pop"); ysr@777: assert(newOop->is_oop(), "Expected an oop"); ysr@777: assert(bm == NULL || bm->isMarked((HeapWord*)newOop), ysr@777: "only grey objects on this stack"); ysr@777: newOop->oop_iterate(cl); ysr@777: if (yield_after && _cm->do_yield_check()) { tonyp@2973: res = false; tonyp@2973: break; ysr@777: } ysr@777: } ysr@777: debug_only(_drain_in_progress = false); ysr@777: return res; ysr@777: } ysr@777: tonyp@3416: void CMMarkStack::note_start_of_gc() { tonyp@3416: assert(_saved_index == -1, tonyp@3416: "note_start_of_gc()/end_of_gc() bracketed incorrectly"); tonyp@3416: _saved_index = _index; tonyp@3416: } tonyp@3416: tonyp@3416: void CMMarkStack::note_end_of_gc() { tonyp@3416: // This is intentionally a guarantee, instead of an assert. If we tonyp@3416: // accidentally add something to the mark stack during GC, it tonyp@3416: // will be a correctness issue so it's better if we crash. we'll tonyp@3416: // only check this once per GC anyway, so it won't be a performance tonyp@3416: // issue in any way. tonyp@3416: guarantee(_saved_index == _index, tonyp@3416: err_msg("saved index: %d index: %d", _saved_index, _index)); tonyp@3416: _saved_index = -1; tonyp@3416: } tonyp@3416: ysr@777: void CMMarkStack::oops_do(OopClosure* f) { tonyp@3416: assert(_saved_index == _index, tonyp@3416: err_msg("saved index: %d index: %d", _saved_index, _index)); tonyp@3416: for (int i = 0; i < _index; i += 1) { ysr@777: f->do_oop(&_base[i]); ysr@777: } ysr@777: } ysr@777: ysr@777: bool ConcurrentMark::not_yet_marked(oop obj) const { coleenp@4037: return _g1h->is_obj_ill(obj); ysr@777: } ysr@777: tonyp@3464: CMRootRegions::CMRootRegions() : tonyp@3464: _young_list(NULL), _cm(NULL), _scan_in_progress(false), tonyp@3464: _should_abort(false), _next_survivor(NULL) { } tonyp@3464: tonyp@3464: void CMRootRegions::init(G1CollectedHeap* g1h, ConcurrentMark* cm) { tonyp@3464: _young_list = g1h->young_list(); tonyp@3464: _cm = cm; tonyp@3464: } tonyp@3464: tonyp@3464: void CMRootRegions::prepare_for_scan() { tonyp@3464: assert(!scan_in_progress(), "pre-condition"); tonyp@3464: tonyp@3464: // Currently, only survivors can be root regions. tonyp@3464: assert(_next_survivor == NULL, "pre-condition"); tonyp@3464: _next_survivor = _young_list->first_survivor_region(); tonyp@3464: _scan_in_progress = (_next_survivor != NULL); tonyp@3464: _should_abort = false; tonyp@3464: } tonyp@3464: tonyp@3464: HeapRegion* CMRootRegions::claim_next() { tonyp@3464: if (_should_abort) { tonyp@3464: // If someone has set the should_abort flag, we return NULL to tonyp@3464: // force the caller to bail out of their loop. tonyp@3464: return NULL; tonyp@3464: } tonyp@3464: tonyp@3464: // Currently, only survivors can be root regions. tonyp@3464: HeapRegion* res = _next_survivor; tonyp@3464: if (res != NULL) { tonyp@3464: MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); tonyp@3464: // Read it again in case it changed while we were waiting for the lock. tonyp@3464: res = _next_survivor; tonyp@3464: if (res != NULL) { tonyp@3464: if (res == _young_list->last_survivor_region()) { tonyp@3464: // We just claimed the last survivor so store NULL to indicate tonyp@3464: // that we're done. tonyp@3464: _next_survivor = NULL; tonyp@3464: } else { tonyp@3464: _next_survivor = res->get_next_young_region(); tonyp@3464: } tonyp@3464: } else { tonyp@3464: // Someone else claimed the last survivor while we were trying tonyp@3464: // to take the lock so nothing else to do. tonyp@3464: } tonyp@3464: } tonyp@3464: assert(res == NULL || res->is_survivor(), "post-condition"); tonyp@3464: tonyp@3464: return res; tonyp@3464: } tonyp@3464: tonyp@3464: void CMRootRegions::scan_finished() { tonyp@3464: assert(scan_in_progress(), "pre-condition"); tonyp@3464: tonyp@3464: // Currently, only survivors can be root regions. tonyp@3464: if (!_should_abort) { tonyp@3464: assert(_next_survivor == NULL, "we should have claimed all survivors"); tonyp@3464: } tonyp@3464: _next_survivor = NULL; tonyp@3464: tonyp@3464: { tonyp@3464: MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); tonyp@3464: _scan_in_progress = false; tonyp@3464: RootRegionScan_lock->notify_all(); tonyp@3464: } tonyp@3464: } tonyp@3464: tonyp@3464: bool CMRootRegions::wait_until_scan_finished() { tonyp@3464: if (!scan_in_progress()) return false; tonyp@3464: tonyp@3464: { tonyp@3464: MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); tonyp@3464: while (scan_in_progress()) { tonyp@3464: RootRegionScan_lock->wait(Mutex::_no_safepoint_check_flag); tonyp@3464: } tonyp@3464: } tonyp@3464: return true; tonyp@3464: } tonyp@3464: ysr@777: #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away ysr@777: #pragma warning( disable:4355 ) // 'this' : used in base member initializer list ysr@777: #endif // _MSC_VER ysr@777: jmasa@3357: uint ConcurrentMark::scale_parallel_threads(uint n_par_threads) { jmasa@3357: return MAX2((n_par_threads + 2) / 4, 1U); jmasa@3294: } jmasa@3294: johnc@4333: ConcurrentMark::ConcurrentMark(G1CollectedHeap* g1h, ReservedSpace heap_rs) : johnc@4333: _g1h(g1h), tschatzl@5697: _markBitMap1(log2_intptr(MinObjAlignment)), tschatzl@5697: _markBitMap2(log2_intptr(MinObjAlignment)), ysr@777: _parallel_marking_threads(0), jmasa@3294: _max_parallel_marking_threads(0), ysr@777: _sleep_factor(0.0), ysr@777: _marking_task_overhead(1.0), ysr@777: _cleanup_sleep_factor(0.0), ysr@777: _cleanup_task_overhead(1.0), tonyp@2472: _cleanup_list("Cleanup List"), johnc@4333: _region_bm((BitMap::idx_t)(g1h->max_regions()), false /* in_resource_area*/), johnc@4333: _card_bm((heap_rs.size() + CardTableModRefBS::card_size - 1) >> johnc@4333: CardTableModRefBS::card_shift, johnc@4333: false /* in_resource_area*/), johnc@3463: ysr@777: _prevMarkBitMap(&_markBitMap1), ysr@777: _nextMarkBitMap(&_markBitMap2), ysr@777: ysr@777: _markStack(this), ysr@777: // _finger set in set_non_marking_state ysr@777: johnc@4173: _max_worker_id(MAX2((uint)ParallelGCThreads, 1U)), ysr@777: // _active_tasks set in set_non_marking_state ysr@777: // _tasks set inside the constructor johnc@4173: _task_queues(new CMTaskQueueSet((int) _max_worker_id)), johnc@4173: _terminator(ParallelTaskTerminator((int) _max_worker_id, _task_queues)), ysr@777: ysr@777: _has_overflown(false), ysr@777: _concurrent(false), tonyp@1054: _has_aborted(false), tonyp@1054: _restart_for_overflow(false), tonyp@1054: _concurrent_marking_in_progress(false), ysr@777: ysr@777: // _verbose_level set below ysr@777: ysr@777: _init_times(), ysr@777: _remark_times(), _remark_mark_times(), _remark_weak_ref_times(), ysr@777: _cleanup_times(), ysr@777: _total_counting_time(0.0), ysr@777: _total_rs_scrub_time(0.0), johnc@3463: johnc@3463: _parallel_workers(NULL), johnc@3463: johnc@3463: _count_card_bitmaps(NULL), johnc@4333: _count_marked_bytes(NULL), johnc@4333: _completed_initialization(false) { tonyp@2973: CMVerboseLevel verbose_level = (CMVerboseLevel) G1MarkingVerboseLevel; tonyp@2973: if (verbose_level < no_verbose) { ysr@777: verbose_level = no_verbose; tonyp@2973: } tonyp@2973: if (verbose_level > high_verbose) { ysr@777: verbose_level = high_verbose; tonyp@2973: } ysr@777: _verbose_level = verbose_level; ysr@777: tonyp@2973: if (verbose_low()) { ysr@777: gclog_or_tty->print_cr("[global] init, heap start = "PTR_FORMAT", " ysr@777: "heap end = "PTR_FORMAT, _heap_start, _heap_end); tonyp@2973: } ysr@777: johnc@4333: if (!_markBitMap1.allocate(heap_rs)) { johnc@4333: warning("Failed to allocate first CM bit map"); johnc@4333: return; johnc@4333: } johnc@4333: if (!_markBitMap2.allocate(heap_rs)) { johnc@4333: warning("Failed to allocate second CM bit map"); johnc@4333: return; johnc@4333: } ysr@777: ysr@777: // Create & start a ConcurrentMark thread. ysr@1280: _cmThread = new ConcurrentMarkThread(this); ysr@1280: assert(cmThread() != NULL, "CM Thread should have been created"); ysr@1280: assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm"); ysr@1280: ysr@777: assert(CGC_lock != NULL, "Where's the CGC_lock?"); johnc@4333: assert(_markBitMap1.covers(heap_rs), "_markBitMap1 inconsistency"); johnc@4333: assert(_markBitMap2.covers(heap_rs), "_markBitMap2 inconsistency"); ysr@777: ysr@777: SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); tonyp@1717: satb_qs.set_buffer_size(G1SATBBufferSize); ysr@777: tonyp@3464: _root_regions.init(_g1h, this); tonyp@3464: jmasa@1719: if (ConcGCThreads > ParallelGCThreads) { johnc@4333: warning("Can't have more ConcGCThreads (" UINT32_FORMAT ") " johnc@4333: "than ParallelGCThreads (" UINT32_FORMAT ").", johnc@4333: ConcGCThreads, ParallelGCThreads); johnc@4333: return; ysr@777: } ysr@777: if (ParallelGCThreads == 0) { ysr@777: // if we are not running with any parallel GC threads we will not ysr@777: // spawn any marking threads either jmasa@3294: _parallel_marking_threads = 0; jmasa@3294: _max_parallel_marking_threads = 0; jmasa@3294: _sleep_factor = 0.0; jmasa@3294: _marking_task_overhead = 1.0; ysr@777: } else { johnc@4547: if (!FLAG_IS_DEFAULT(ConcGCThreads) && ConcGCThreads > 0) { johnc@4547: // Note: ConcGCThreads has precedence over G1MarkingOverheadPercent ysr@777: // if both are set ysr@777: _sleep_factor = 0.0; ysr@777: _marking_task_overhead = 1.0; johnc@1186: } else if (G1MarkingOverheadPercent > 0) { johnc@4547: // We will calculate the number of parallel marking threads based johnc@4547: // on a target overhead with respect to the soft real-time goal johnc@1186: double marking_overhead = (double) G1MarkingOverheadPercent / 100.0; ysr@777: double overall_cm_overhead = johnc@1186: (double) MaxGCPauseMillis * marking_overhead / johnc@1186: (double) GCPauseIntervalMillis; ysr@777: double cpu_ratio = 1.0 / (double) os::processor_count(); ysr@777: double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio); ysr@777: double marking_task_overhead = ysr@777: overall_cm_overhead / marking_thread_num * ysr@777: (double) os::processor_count(); ysr@777: double sleep_factor = ysr@777: (1.0 - marking_task_overhead) / marking_task_overhead; ysr@777: johnc@4547: FLAG_SET_ERGO(uintx, ConcGCThreads, (uint) marking_thread_num); ysr@777: _sleep_factor = sleep_factor; ysr@777: _marking_task_overhead = marking_task_overhead; ysr@777: } else { johnc@4547: // Calculate the number of parallel marking threads by scaling johnc@4547: // the number of parallel GC threads. johnc@4547: uint marking_thread_num = scale_parallel_threads((uint) ParallelGCThreads); johnc@4547: FLAG_SET_ERGO(uintx, ConcGCThreads, marking_thread_num); ysr@777: _sleep_factor = 0.0; ysr@777: _marking_task_overhead = 1.0; ysr@777: } ysr@777: johnc@4547: assert(ConcGCThreads > 0, "Should have been set"); johnc@4547: _parallel_marking_threads = (uint) ConcGCThreads; johnc@4547: _max_parallel_marking_threads = _parallel_marking_threads; johnc@4547: tonyp@2973: if (parallel_marking_threads() > 1) { ysr@777: _cleanup_task_overhead = 1.0; tonyp@2973: } else { ysr@777: _cleanup_task_overhead = marking_task_overhead(); tonyp@2973: } ysr@777: _cleanup_sleep_factor = ysr@777: (1.0 - cleanup_task_overhead()) / cleanup_task_overhead(); ysr@777: ysr@777: #if 0 ysr@777: gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads()); ysr@777: gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead()); ysr@777: gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor()); ysr@777: gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead()); ysr@777: gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor()); ysr@777: #endif ysr@777: tonyp@1458: guarantee(parallel_marking_threads() > 0, "peace of mind"); jmasa@2188: _parallel_workers = new FlexibleWorkGang("G1 Parallel Marking Threads", jmasa@3357: _max_parallel_marking_threads, false, true); jmasa@2188: if (_parallel_workers == NULL) { ysr@777: vm_exit_during_initialization("Failed necessary allocation."); jmasa@2188: } else { jmasa@2188: _parallel_workers->initialize_workers(); jmasa@2188: } ysr@777: } ysr@777: johnc@4333: if (FLAG_IS_DEFAULT(MarkStackSize)) { johnc@4333: uintx mark_stack_size = johnc@4333: MIN2(MarkStackSizeMax, johnc@4333: MAX2(MarkStackSize, (uintx) (parallel_marking_threads() * TASKQUEUE_SIZE))); johnc@4333: // Verify that the calculated value for MarkStackSize is in range. johnc@4333: // It would be nice to use the private utility routine from Arguments. johnc@4333: if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) { johnc@4333: warning("Invalid value calculated for MarkStackSize (" UINTX_FORMAT "): " johnc@4333: "must be between " UINTX_FORMAT " and " UINTX_FORMAT, johnc@4333: mark_stack_size, 1, MarkStackSizeMax); johnc@4333: return; johnc@4333: } johnc@4333: FLAG_SET_ERGO(uintx, MarkStackSize, mark_stack_size); johnc@4333: } else { johnc@4333: // Verify MarkStackSize is in range. johnc@4333: if (FLAG_IS_CMDLINE(MarkStackSize)) { johnc@4333: if (FLAG_IS_DEFAULT(MarkStackSizeMax)) { johnc@4333: if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) { johnc@4333: warning("Invalid value specified for MarkStackSize (" UINTX_FORMAT "): " johnc@4333: "must be between " UINTX_FORMAT " and " UINTX_FORMAT, johnc@4333: MarkStackSize, 1, MarkStackSizeMax); johnc@4333: return; johnc@4333: } johnc@4333: } else if (FLAG_IS_CMDLINE(MarkStackSizeMax)) { johnc@4333: if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) { johnc@4333: warning("Invalid value specified for MarkStackSize (" UINTX_FORMAT ")" johnc@4333: " or for MarkStackSizeMax (" UINTX_FORMAT ")", johnc@4333: MarkStackSize, MarkStackSizeMax); johnc@4333: return; johnc@4333: } johnc@4333: } johnc@4333: } johnc@4333: } johnc@4333: johnc@4333: if (!_markStack.allocate(MarkStackSize)) { johnc@4333: warning("Failed to allocate CM marking stack"); johnc@4333: return; johnc@4333: } johnc@4333: johnc@4333: _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_worker_id, mtGC); johnc@4333: _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_worker_id, mtGC); johnc@4333: johnc@4333: _count_card_bitmaps = NEW_C_HEAP_ARRAY(BitMap, _max_worker_id, mtGC); johnc@4333: _count_marked_bytes = NEW_C_HEAP_ARRAY(size_t*, _max_worker_id, mtGC); johnc@4333: johnc@4333: BitMap::idx_t card_bm_size = _card_bm.size(); johnc@4333: johnc@4333: // so that the assertion in MarkingTaskQueue::task_queue doesn't fail johnc@4333: _active_tasks = _max_worker_id; johnc@4333: johnc@4333: size_t max_regions = (size_t) _g1h->max_regions(); johnc@4333: for (uint i = 0; i < _max_worker_id; ++i) { johnc@4333: CMTaskQueue* task_queue = new CMTaskQueue(); johnc@4333: task_queue->initialize(); johnc@4333: _task_queues->register_queue(i, task_queue); johnc@4333: johnc@4333: _count_card_bitmaps[i] = BitMap(card_bm_size, false); johnc@4333: _count_marked_bytes[i] = NEW_C_HEAP_ARRAY(size_t, max_regions, mtGC); johnc@4333: johnc@4333: _tasks[i] = new CMTask(i, this, johnc@4333: _count_marked_bytes[i], johnc@4333: &_count_card_bitmaps[i], johnc@4333: task_queue, _task_queues); johnc@4333: johnc@4333: _accum_task_vtime[i] = 0.0; johnc@4333: } johnc@4333: johnc@4333: // Calculate the card number for the bottom of the heap. Used johnc@4333: // in biasing indexes into the accounting card bitmaps. johnc@4333: _heap_bottom_card_num = johnc@4333: intptr_t(uintptr_t(_g1h->reserved_region().start()) >> johnc@4333: CardTableModRefBS::card_shift); johnc@4333: johnc@4333: // Clear all the liveness counting data johnc@4333: clear_all_count_data(); johnc@4333: ysr@777: // so that the call below can read a sensible value johnc@4333: _heap_start = (HeapWord*) heap_rs.base(); ysr@777: set_non_marking_state(); johnc@4333: _completed_initialization = true; ysr@777: } ysr@777: ysr@777: void ConcurrentMark::update_g1_committed(bool force) { ysr@777: // If concurrent marking is not in progress, then we do not need to tonyp@3691: // update _heap_end. tonyp@2973: if (!concurrent_marking_in_progress() && !force) return; ysr@777: ysr@777: MemRegion committed = _g1h->g1_committed(); tonyp@1458: assert(committed.start() == _heap_start, "start shouldn't change"); ysr@777: HeapWord* new_end = committed.end(); ysr@777: if (new_end > _heap_end) { ysr@777: // The heap has been expanded. ysr@777: ysr@777: _heap_end = new_end; ysr@777: } ysr@777: // Notice that the heap can also shrink. However, this only happens ysr@777: // during a Full GC (at least currently) and the entire marking ysr@777: // phase will bail out and the task will not be restarted. So, let's ysr@777: // do nothing. ysr@777: } ysr@777: ysr@777: void ConcurrentMark::reset() { ysr@777: // Starting values for these two. This should be called in a STW ysr@777: // phase. CM will be notified of any future g1_committed expansions ysr@777: // will be at the end of evacuation pauses, when tasks are ysr@777: // inactive. ysr@777: MemRegion committed = _g1h->g1_committed(); ysr@777: _heap_start = committed.start(); ysr@777: _heap_end = committed.end(); ysr@777: tonyp@1458: // Separated the asserts so that we know which one fires. tonyp@1458: assert(_heap_start != NULL, "heap bounds should look ok"); tonyp@1458: assert(_heap_end != NULL, "heap bounds should look ok"); tonyp@1458: assert(_heap_start < _heap_end, "heap bounds should look ok"); ysr@777: johnc@4386: // Reset all the marking data structures and any necessary flags johnc@4386: reset_marking_state(); ysr@777: tonyp@2973: if (verbose_low()) { ysr@777: gclog_or_tty->print_cr("[global] resetting"); tonyp@2973: } ysr@777: ysr@777: // We do reset all of them, since different phases will use ysr@777: // different number of active threads. So, it's easiest to have all ysr@777: // of them ready. johnc@4173: for (uint i = 0; i < _max_worker_id; ++i) { ysr@777: _tasks[i]->reset(_nextMarkBitMap); johnc@2190: } ysr@777: ysr@777: // we need this to make sure that the flag is on during the evac ysr@777: // pause with initial mark piggy-backed ysr@777: set_concurrent_marking_in_progress(); ysr@777: } ysr@777: johnc@4386: johnc@4386: void ConcurrentMark::reset_marking_state(bool clear_overflow) { johnc@4386: _markStack.set_should_expand(); johnc@4386: _markStack.setEmpty(); // Also clears the _markStack overflow flag johnc@4386: if (clear_overflow) { johnc@4386: clear_has_overflown(); johnc@4386: } else { johnc@4386: assert(has_overflown(), "pre-condition"); johnc@4386: } johnc@4386: _finger = _heap_start; johnc@4386: johnc@4386: for (uint i = 0; i < _max_worker_id; ++i) { johnc@4386: CMTaskQueue* queue = _task_queues->queue(i); johnc@4386: queue->set_empty(); johnc@4386: } johnc@4386: } johnc@4386: johnc@4788: void ConcurrentMark::set_concurrency(uint active_tasks) { johnc@4173: assert(active_tasks <= _max_worker_id, "we should not have more"); ysr@777: ysr@777: _active_tasks = active_tasks; ysr@777: // Need to update the three data structures below according to the ysr@777: // number of active threads for this phase. ysr@777: _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues); ysr@777: _first_overflow_barrier_sync.set_n_workers((int) active_tasks); ysr@777: _second_overflow_barrier_sync.set_n_workers((int) active_tasks); johnc@4788: } johnc@4788: johnc@4788: void ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurrent) { johnc@4788: set_concurrency(active_tasks); ysr@777: ysr@777: _concurrent = concurrent; ysr@777: // We propagate this to all tasks, not just the active ones. johnc@4173: for (uint i = 0; i < _max_worker_id; ++i) ysr@777: _tasks[i]->set_concurrent(concurrent); ysr@777: ysr@777: if (concurrent) { ysr@777: set_concurrent_marking_in_progress(); ysr@777: } else { ysr@777: // We currently assume that the concurrent flag has been set to ysr@777: // false before we start remark. At this point we should also be ysr@777: // in a STW phase. tonyp@1458: assert(!concurrent_marking_in_progress(), "invariant"); johnc@4788: assert(_finger == _heap_end, johnc@4788: err_msg("only way to get here: _finger: "PTR_FORMAT", _heap_end: "PTR_FORMAT, johnc@4788: _finger, _heap_end)); ysr@777: update_g1_committed(true); ysr@777: } ysr@777: } ysr@777: ysr@777: void ConcurrentMark::set_non_marking_state() { ysr@777: // We set the global marking state to some default values when we're ysr@777: // not doing marking. johnc@4386: reset_marking_state(); ysr@777: _active_tasks = 0; ysr@777: clear_concurrent_marking_in_progress(); ysr@777: } ysr@777: ysr@777: ConcurrentMark::~ConcurrentMark() { stefank@3364: // The ConcurrentMark instance is never freed. stefank@3364: ShouldNotReachHere(); ysr@777: } ysr@777: ysr@777: void ConcurrentMark::clearNextBitmap() { tonyp@1794: G1CollectedHeap* g1h = G1CollectedHeap::heap(); tonyp@1794: G1CollectorPolicy* g1p = g1h->g1_policy(); tonyp@1794: tonyp@1794: // Make sure that the concurrent mark thread looks to still be in tonyp@1794: // the current cycle. tonyp@1794: guarantee(cmThread()->during_cycle(), "invariant"); tonyp@1794: tonyp@1794: // We are finishing up the current cycle by clearing the next tonyp@1794: // marking bitmap and getting it ready for the next cycle. During tonyp@1794: // this time no other cycle can start. So, let's make sure that this tonyp@1794: // is the case. tonyp@1794: guarantee(!g1h->mark_in_progress(), "invariant"); tonyp@1794: tonyp@1794: // clear the mark bitmap (no grey objects to start with). tonyp@1794: // We need to do this in chunks and offer to yield in between tonyp@1794: // each chunk. tonyp@1794: HeapWord* start = _nextMarkBitMap->startWord(); tonyp@1794: HeapWord* end = _nextMarkBitMap->endWord(); tonyp@1794: HeapWord* cur = start; tonyp@1794: size_t chunkSize = M; tonyp@1794: while (cur < end) { tonyp@1794: HeapWord* next = cur + chunkSize; tonyp@2973: if (next > end) { tonyp@1794: next = end; tonyp@2973: } tonyp@1794: MemRegion mr(cur,next); tonyp@1794: _nextMarkBitMap->clearRange(mr); tonyp@1794: cur = next; tonyp@1794: do_yield_check(); tonyp@1794: tonyp@1794: // Repeat the asserts from above. We'll do them as asserts here to tonyp@1794: // minimize their overhead on the product. However, we'll have tonyp@1794: // them as guarantees at the beginning / end of the bitmap tonyp@1794: // clearing to get some checking in the product. tonyp@1794: assert(cmThread()->during_cycle(), "invariant"); tonyp@1794: assert(!g1h->mark_in_progress(), "invariant"); tonyp@1794: } tonyp@1794: johnc@3463: // Clear the liveness counting data johnc@3463: clear_all_count_data(); johnc@3463: tonyp@1794: // Repeat the asserts from above. tonyp@1794: guarantee(cmThread()->during_cycle(), "invariant"); tonyp@1794: guarantee(!g1h->mark_in_progress(), "invariant"); ysr@777: } ysr@777: ysr@777: class NoteStartOfMarkHRClosure: public HeapRegionClosure { ysr@777: public: ysr@777: bool doHeapRegion(HeapRegion* r) { ysr@777: if (!r->continuesHumongous()) { tonyp@3416: r->note_start_of_marking(); ysr@777: } ysr@777: return false; ysr@777: } ysr@777: }; ysr@777: ysr@777: void ConcurrentMark::checkpointRootsInitialPre() { ysr@777: G1CollectedHeap* g1h = G1CollectedHeap::heap(); ysr@777: G1CollectorPolicy* g1p = g1h->g1_policy(); ysr@777: ysr@777: _has_aborted = false; ysr@777: jcoomes@1902: #ifndef PRODUCT tonyp@1479: if (G1PrintReachableAtInitialMark) { tonyp@1823: print_reachable("at-cycle-start", johnc@2969: VerifyOption_G1UsePrevMarking, true /* all */); tonyp@1479: } jcoomes@1902: #endif ysr@777: ysr@777: // Initialise marking structures. This has to be done in a STW phase. ysr@777: reset(); tonyp@3416: tonyp@3416: // For each region note start of marking. tonyp@3416: NoteStartOfMarkHRClosure startcl; tonyp@3416: g1h->heap_region_iterate(&startcl); ysr@777: } ysr@777: ysr@777: ysr@777: void ConcurrentMark::checkpointRootsInitialPost() { ysr@777: G1CollectedHeap* g1h = G1CollectedHeap::heap(); ysr@777: tonyp@2848: // If we force an overflow during remark, the remark operation will tonyp@2848: // actually abort and we'll restart concurrent marking. If we always tonyp@2848: // force an oveflow during remark we'll never actually complete the tonyp@2848: // marking phase. So, we initilize this here, at the start of the tonyp@2848: // cycle, so that at the remaining overflow number will decrease at tonyp@2848: // every remark and we'll eventually not need to cause one. tonyp@2848: force_overflow_stw()->init(); tonyp@2848: johnc@3175: // Start Concurrent Marking weak-reference discovery. johnc@3175: ReferenceProcessor* rp = g1h->ref_processor_cm(); johnc@3175: // enable ("weak") refs discovery johnc@3175: rp->enable_discovery(true /*verify_disabled*/, true /*verify_no_refs*/); ysr@892: rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle ysr@777: ysr@777: SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); tonyp@1752: // This is the start of the marking cycle, we're expected all tonyp@1752: // threads to have SATB queues with active set to false. tonyp@1752: satb_mq_set.set_active_all_threads(true, /* new active value */ tonyp@1752: false /* expected_active */); ysr@777: tonyp@3464: _root_regions.prepare_for_scan(); tonyp@3464: ysr@777: // update_g1_committed() will be called at the end of an evac pause ysr@777: // when marking is on. So, it's also called at the end of the ysr@777: // initial-mark pause to update the heap end, if the heap expands ysr@777: // during it. No need to call it here. ysr@777: } ysr@777: ysr@777: /* tonyp@2848: * Notice that in the next two methods, we actually leave the STS tonyp@2848: * during the barrier sync and join it immediately afterwards. If we tonyp@2848: * do not do this, the following deadlock can occur: one thread could tonyp@2848: * be in the barrier sync code, waiting for the other thread to also tonyp@2848: * sync up, whereas another one could be trying to yield, while also tonyp@2848: * waiting for the other threads to sync up too. tonyp@2848: * tonyp@2848: * Note, however, that this code is also used during remark and in tonyp@2848: * this case we should not attempt to leave / enter the STS, otherwise tonyp@2848: * we'll either hit an asseert (debug / fastdebug) or deadlock tonyp@2848: * (product). So we should only leave / enter the STS if we are tonyp@2848: * operating concurrently. tonyp@2848: * tonyp@2848: * Because the thread that does the sync barrier has left the STS, it tonyp@2848: * is possible to be suspended for a Full GC or an evacuation pause tonyp@2848: * could occur. This is actually safe, since the entering the sync tonyp@2848: * barrier is one of the last things do_marking_step() does, and it tonyp@2848: * doesn't manipulate any data structures afterwards. tonyp@2848: */ ysr@777: johnc@4173: void ConcurrentMark::enter_first_sync_barrier(uint worker_id) { tonyp@2973: if (verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] entering first barrier", worker_id); tonyp@2973: } ysr@777: tonyp@2848: if (concurrent()) { tonyp@2848: ConcurrentGCThread::stsLeave(); tonyp@2848: } ysr@777: _first_overflow_barrier_sync.enter(); tonyp@2848: if (concurrent()) { tonyp@2848: ConcurrentGCThread::stsJoin(); tonyp@2848: } ysr@777: // at this point everyone should have synced up and not be doing any ysr@777: // more work ysr@777: tonyp@2973: if (verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] leaving first barrier", worker_id); tonyp@2973: } ysr@777: johnc@4788: // If we're executing the concurrent phase of marking, reset the marking johnc@4788: // state; otherwise the marking state is reset after reference processing, johnc@4788: // during the remark pause. johnc@4788: // If we reset here as a result of an overflow during the remark we will johnc@4788: // see assertion failures from any subsequent set_concurrency_and_phase() johnc@4788: // calls. johnc@4788: if (concurrent()) { johnc@4788: // let the task associated with with worker 0 do this johnc@4788: if (worker_id == 0) { johnc@4788: // task 0 is responsible for clearing the global data structures johnc@4788: // We should be here because of an overflow. During STW we should johnc@4788: // not clear the overflow flag since we rely on it being true when johnc@4788: // we exit this method to abort the pause and restart concurent johnc@4788: // marking. johnc@4788: reset_marking_state(true /* clear_overflow */); johnc@4788: force_overflow()->update(); johnc@4788: johnc@4788: if (G1Log::fine()) { johnc@4788: gclog_or_tty->date_stamp(PrintGCDateStamps); johnc@4788: gclog_or_tty->stamp(PrintGCTimeStamps); johnc@4788: gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]"); johnc@4788: } ysr@777: } ysr@777: } ysr@777: ysr@777: // after this, each task should reset its own data structures then ysr@777: // then go into the second barrier ysr@777: } ysr@777: johnc@4173: void ConcurrentMark::enter_second_sync_barrier(uint worker_id) { tonyp@2973: if (verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] entering second barrier", worker_id); tonyp@2973: } ysr@777: tonyp@2848: if (concurrent()) { tonyp@2848: ConcurrentGCThread::stsLeave(); tonyp@2848: } ysr@777: _second_overflow_barrier_sync.enter(); tonyp@2848: if (concurrent()) { tonyp@2848: ConcurrentGCThread::stsJoin(); tonyp@2848: } johnc@4788: // at this point everything should be re-initialized and ready to go ysr@777: tonyp@2973: if (verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] leaving second barrier", worker_id); tonyp@2973: } ysr@777: } ysr@777: tonyp@2848: #ifndef PRODUCT tonyp@2848: void ForceOverflowSettings::init() { tonyp@2848: _num_remaining = G1ConcMarkForceOverflow; tonyp@2848: _force = false; tonyp@2848: update(); tonyp@2848: } tonyp@2848: tonyp@2848: void ForceOverflowSettings::update() { tonyp@2848: if (_num_remaining > 0) { tonyp@2848: _num_remaining -= 1; tonyp@2848: _force = true; tonyp@2848: } else { tonyp@2848: _force = false; tonyp@2848: } tonyp@2848: } tonyp@2848: tonyp@2848: bool ForceOverflowSettings::should_force() { tonyp@2848: if (_force) { tonyp@2848: _force = false; tonyp@2848: return true; tonyp@2848: } else { tonyp@2848: return false; tonyp@2848: } tonyp@2848: } tonyp@2848: #endif // !PRODUCT tonyp@2848: ysr@777: class CMConcurrentMarkingTask: public AbstractGangTask { ysr@777: private: ysr@777: ConcurrentMark* _cm; ysr@777: ConcurrentMarkThread* _cmt; ysr@777: ysr@777: public: jmasa@3357: void work(uint worker_id) { tonyp@1458: assert(Thread::current()->is_ConcurrentGC_thread(), tonyp@1458: "this should only be done by a conc GC thread"); johnc@2316: ResourceMark rm; ysr@777: ysr@777: double start_vtime = os::elapsedVTime(); ysr@777: ysr@777: ConcurrentGCThread::stsJoin(); ysr@777: jmasa@3357: assert(worker_id < _cm->active_tasks(), "invariant"); jmasa@3357: CMTask* the_task = _cm->task(worker_id); ysr@777: the_task->record_start_time(); ysr@777: if (!_cm->has_aborted()) { ysr@777: do { ysr@777: double start_vtime_sec = os::elapsedVTime(); ysr@777: double start_time_sec = os::elapsedTime(); johnc@2494: double mark_step_duration_ms = G1ConcMarkStepDurationMillis; johnc@2494: johnc@2494: the_task->do_marking_step(mark_step_duration_ms, johnc@4787: true /* do_termination */, johnc@4787: false /* is_serial*/); johnc@2494: ysr@777: double end_time_sec = os::elapsedTime(); ysr@777: double end_vtime_sec = os::elapsedVTime(); ysr@777: double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec; ysr@777: double elapsed_time_sec = end_time_sec - start_time_sec; ysr@777: _cm->clear_has_overflown(); ysr@777: jmasa@3357: bool ret = _cm->do_yield_check(worker_id); ysr@777: ysr@777: jlong sleep_time_ms; ysr@777: if (!_cm->has_aborted() && the_task->has_aborted()) { ysr@777: sleep_time_ms = ysr@777: (jlong) (elapsed_vtime_sec * _cm->sleep_factor() * 1000.0); ysr@777: ConcurrentGCThread::stsLeave(); ysr@777: os::sleep(Thread::current(), sleep_time_ms, false); ysr@777: ConcurrentGCThread::stsJoin(); ysr@777: } ysr@777: double end_time2_sec = os::elapsedTime(); ysr@777: double elapsed_time2_sec = end_time2_sec - start_time_sec; ysr@777: ysr@777: #if 0 ysr@777: gclog_or_tty->print_cr("CM: elapsed %1.4lf ms, sleep %1.4lf ms, " ysr@777: "overhead %1.4lf", ysr@777: elapsed_vtime_sec * 1000.0, (double) sleep_time_ms, ysr@777: the_task->conc_overhead(os::elapsedTime()) * 8.0); ysr@777: gclog_or_tty->print_cr("elapsed time %1.4lf ms, time 2: %1.4lf ms", ysr@777: elapsed_time_sec * 1000.0, elapsed_time2_sec * 1000.0); ysr@777: #endif ysr@777: } while (!_cm->has_aborted() && the_task->has_aborted()); ysr@777: } ysr@777: the_task->record_end_time(); tonyp@1458: guarantee(!the_task->has_aborted() || _cm->has_aborted(), "invariant"); ysr@777: ysr@777: ConcurrentGCThread::stsLeave(); ysr@777: ysr@777: double end_vtime = os::elapsedVTime(); jmasa@3357: _cm->update_accum_task_vtime(worker_id, end_vtime - start_vtime); ysr@777: } ysr@777: ysr@777: CMConcurrentMarkingTask(ConcurrentMark* cm, ysr@777: ConcurrentMarkThread* cmt) : ysr@777: AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { } ysr@777: ysr@777: ~CMConcurrentMarkingTask() { } ysr@777: }; ysr@777: jmasa@3294: // Calculates the number of active workers for a concurrent jmasa@3294: // phase. jmasa@3357: uint ConcurrentMark::calc_parallel_marking_threads() { johnc@3338: if (G1CollectedHeap::use_parallel_gc_threads()) { jmasa@3357: uint n_conc_workers = 0; jmasa@3294: if (!UseDynamicNumberOfGCThreads || jmasa@3294: (!FLAG_IS_DEFAULT(ConcGCThreads) && jmasa@3294: !ForceDynamicNumberOfGCThreads)) { jmasa@3294: n_conc_workers = max_parallel_marking_threads(); jmasa@3294: } else { jmasa@3294: n_conc_workers = jmasa@3294: AdaptiveSizePolicy::calc_default_active_workers( jmasa@3294: max_parallel_marking_threads(), jmasa@3294: 1, /* Minimum workers */ jmasa@3294: parallel_marking_threads(), jmasa@3294: Threads::number_of_non_daemon_threads()); jmasa@3294: // Don't scale down "n_conc_workers" by scale_parallel_threads() because jmasa@3294: // that scaling has already gone into "_max_parallel_marking_threads". jmasa@3294: } johnc@3338: assert(n_conc_workers > 0, "Always need at least 1"); johnc@3338: return n_conc_workers; jmasa@3294: } johnc@3338: // If we are not running with any parallel GC threads we will not johnc@3338: // have spawned any marking threads either. Hence the number of johnc@3338: // concurrent workers should be 0. johnc@3338: return 0; jmasa@3294: } jmasa@3294: tonyp@3464: void ConcurrentMark::scanRootRegion(HeapRegion* hr, uint worker_id) { tonyp@3464: // Currently, only survivors can be root regions. tonyp@3464: assert(hr->next_top_at_mark_start() == hr->bottom(), "invariant"); tonyp@3464: G1RootRegionScanClosure cl(_g1h, this, worker_id); tonyp@3464: tonyp@3464: const uintx interval = PrefetchScanIntervalInBytes; tonyp@3464: HeapWord* curr = hr->bottom(); tonyp@3464: const HeapWord* end = hr->top(); tonyp@3464: while (curr < end) { tonyp@3464: Prefetch::read(curr, interval); tonyp@3464: oop obj = oop(curr); tonyp@3464: int size = obj->oop_iterate(&cl); tonyp@3464: assert(size == obj->size(), "sanity"); tonyp@3464: curr += size; tonyp@3464: } tonyp@3464: } tonyp@3464: tonyp@3464: class CMRootRegionScanTask : public AbstractGangTask { tonyp@3464: private: tonyp@3464: ConcurrentMark* _cm; tonyp@3464: tonyp@3464: public: tonyp@3464: CMRootRegionScanTask(ConcurrentMark* cm) : tonyp@3464: AbstractGangTask("Root Region Scan"), _cm(cm) { } tonyp@3464: tonyp@3464: void work(uint worker_id) { tonyp@3464: assert(Thread::current()->is_ConcurrentGC_thread(), tonyp@3464: "this should only be done by a conc GC thread"); tonyp@3464: tonyp@3464: CMRootRegions* root_regions = _cm->root_regions(); tonyp@3464: HeapRegion* hr = root_regions->claim_next(); tonyp@3464: while (hr != NULL) { tonyp@3464: _cm->scanRootRegion(hr, worker_id); tonyp@3464: hr = root_regions->claim_next(); tonyp@3464: } tonyp@3464: } tonyp@3464: }; tonyp@3464: tonyp@3464: void ConcurrentMark::scanRootRegions() { tonyp@3464: // scan_in_progress() will have been set to true only if there was tonyp@3464: // at least one root region to scan. So, if it's false, we tonyp@3464: // should not attempt to do any further work. tonyp@3464: if (root_regions()->scan_in_progress()) { tonyp@3464: _parallel_marking_threads = calc_parallel_marking_threads(); tonyp@3464: assert(parallel_marking_threads() <= max_parallel_marking_threads(), tonyp@3464: "Maximum number of marking threads exceeded"); tonyp@3464: uint active_workers = MAX2(1U, parallel_marking_threads()); tonyp@3464: tonyp@3464: CMRootRegionScanTask task(this); johnc@4549: if (use_parallel_marking_threads()) { tonyp@3464: _parallel_workers->set_active_workers((int) active_workers); tonyp@3464: _parallel_workers->run_task(&task); tonyp@3464: } else { tonyp@3464: task.work(0); tonyp@3464: } tonyp@3464: tonyp@3464: // It's possible that has_aborted() is true here without actually tonyp@3464: // aborting the survivor scan earlier. This is OK as it's tonyp@3464: // mainly used for sanity checking. tonyp@3464: root_regions()->scan_finished(); tonyp@3464: } tonyp@3464: } tonyp@3464: ysr@777: void ConcurrentMark::markFromRoots() { ysr@777: // we might be tempted to assert that: ysr@777: // assert(asynch == !SafepointSynchronize::is_at_safepoint(), ysr@777: // "inconsistent argument?"); ysr@777: // However that wouldn't be right, because it's possible that ysr@777: // a safepoint is indeed in progress as a younger generation ysr@777: // stop-the-world GC happens even as we mark in this generation. ysr@777: ysr@777: _restart_for_overflow = false; tonyp@2848: force_overflow_conc()->init(); jmasa@3294: jmasa@3294: // _g1h has _n_par_threads jmasa@3294: _parallel_marking_threads = calc_parallel_marking_threads(); jmasa@3294: assert(parallel_marking_threads() <= max_parallel_marking_threads(), jmasa@3294: "Maximum number of marking threads exceeded"); johnc@3338: jmasa@3357: uint active_workers = MAX2(1U, parallel_marking_threads()); johnc@3338: johnc@4788: // Parallel task terminator is set in "set_concurrency_and_phase()" johnc@4788: set_concurrency_and_phase(active_workers, true /* concurrent */); ysr@777: ysr@777: CMConcurrentMarkingTask markingTask(this, cmThread()); johnc@4549: if (use_parallel_marking_threads()) { johnc@3338: _parallel_workers->set_active_workers((int)active_workers); johnc@3338: // Don't set _n_par_threads because it affects MT in proceess_strong_roots() johnc@3338: // and the decisions on that MT processing is made elsewhere. johnc@3338: assert(_parallel_workers->active_workers() > 0, "Should have been set"); ysr@777: _parallel_workers->run_task(&markingTask); tonyp@2973: } else { ysr@777: markingTask.work(0); tonyp@2973: } ysr@777: print_stats(); ysr@777: } ysr@777: ysr@777: void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) { ysr@777: // world is stopped at this checkpoint ysr@777: assert(SafepointSynchronize::is_at_safepoint(), ysr@777: "world should be stopped"); johnc@3175: ysr@777: G1CollectedHeap* g1h = G1CollectedHeap::heap(); ysr@777: ysr@777: // If a full collection has happened, we shouldn't do this. ysr@777: if (has_aborted()) { ysr@777: g1h->set_marking_complete(); // So bitmap clearing isn't confused ysr@777: return; ysr@777: } ysr@777: kamg@2445: SvcGCMarker sgcm(SvcGCMarker::OTHER); kamg@2445: ysr@1280: if (VerifyDuringGC) { ysr@1280: HandleMark hm; // handle scope ysr@1280: Universe::heap()->prepare_for_verify(); stefank@5018: Universe::verify(VerifyOption_G1UsePrevMarking, stefank@5018: " VerifyDuringGC:(before)"); ysr@1280: } ysr@1280: ysr@777: G1CollectorPolicy* g1p = g1h->g1_policy(); ysr@777: g1p->record_concurrent_mark_remark_start(); ysr@777: ysr@777: double start = os::elapsedTime(); ysr@777: ysr@777: checkpointRootsFinalWork(); ysr@777: ysr@777: double mark_work_end = os::elapsedTime(); ysr@777: ysr@777: weakRefsWork(clear_all_soft_refs); ysr@777: ysr@777: if (has_overflown()) { ysr@777: // Oops. We overflowed. Restart concurrent marking. ysr@777: _restart_for_overflow = true; johnc@4789: if (G1TraceMarkStackOverflow) { johnc@4789: gclog_or_tty->print_cr("\nRemark led to restart for overflow."); johnc@4789: } johnc@4789: johnc@4789: // Verify the heap w.r.t. the previous marking bitmap. johnc@4789: if (VerifyDuringGC) { johnc@4789: HandleMark hm; // handle scope johnc@4789: Universe::heap()->prepare_for_verify(); stefank@5018: Universe::verify(VerifyOption_G1UsePrevMarking, stefank@5018: " VerifyDuringGC:(overflow)"); johnc@4789: } johnc@4789: johnc@4386: // Clear the marking state because we will be restarting johnc@4386: // marking due to overflowing the global mark stack. johnc@4386: reset_marking_state(); ysr@777: } else { johnc@3463: // Aggregate the per-task counting data that we have accumulated johnc@3463: // while marking. johnc@3463: aggregate_count_data(); johnc@3463: tonyp@2469: SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); ysr@777: // We're done with marking. tonyp@1752: // This is the end of the marking cycle, we're expected all tonyp@1752: // threads to have SATB queues with active set to true. tonyp@2469: satb_mq_set.set_active_all_threads(false, /* new active value */ tonyp@2469: true /* expected_active */); tonyp@1246: tonyp@1246: if (VerifyDuringGC) { ysr@1280: HandleMark hm; // handle scope ysr@1280: Universe::heap()->prepare_for_verify(); stefank@5018: Universe::verify(VerifyOption_G1UseNextMarking, stefank@5018: " VerifyDuringGC:(after)"); tonyp@1246: } johnc@2494: assert(!restart_for_overflow(), "sanity"); johnc@4386: // Completely reset the marking state since marking completed johnc@4386: set_non_marking_state(); johnc@2494: } johnc@2494: johnc@4333: // Expand the marking stack, if we have to and if we can. johnc@4333: if (_markStack.should_expand()) { johnc@4333: _markStack.expand(); johnc@4333: } johnc@4333: ysr@777: // Statistics ysr@777: double now = os::elapsedTime(); ysr@777: _remark_mark_times.add((mark_work_end - start) * 1000.0); ysr@777: _remark_weak_ref_times.add((now - mark_work_end) * 1000.0); ysr@777: _remark_times.add((now - start) * 1000.0); ysr@777: ysr@777: g1p->record_concurrent_mark_remark_end(); sla@5237: sla@5237: G1CMIsAliveClosure is_alive(g1h); sla@5237: g1h->gc_tracer_cm()->report_object_count_after_gc(&is_alive); ysr@777: } ysr@777: johnc@3731: // Base class of the closures that finalize and verify the johnc@3731: // liveness counting data. johnc@3731: class CMCountDataClosureBase: public HeapRegionClosure { johnc@3731: protected: johnc@4123: G1CollectedHeap* _g1h; ysr@777: ConcurrentMark* _cm; johnc@4123: CardTableModRefBS* _ct_bs; johnc@4123: johnc@3463: BitMap* _region_bm; johnc@3463: BitMap* _card_bm; johnc@3463: johnc@4123: // Takes a region that's not empty (i.e., it has at least one tonyp@1264: // live object in it and sets its corresponding bit on the region tonyp@1264: // bitmap to 1. If the region is "starts humongous" it will also set tonyp@1264: // to 1 the bits on the region bitmap that correspond to its tonyp@1264: // associated "continues humongous" regions. tonyp@1264: void set_bit_for_region(HeapRegion* hr) { tonyp@1264: assert(!hr->continuesHumongous(), "should have filtered those out"); tonyp@1264: tonyp@3713: BitMap::idx_t index = (BitMap::idx_t) hr->hrs_index(); tonyp@1264: if (!hr->startsHumongous()) { tonyp@1264: // Normal (non-humongous) case: just set the bit. tonyp@3713: _region_bm->par_at_put(index, true); tonyp@1264: } else { tonyp@1264: // Starts humongous case: calculate how many regions are part of johnc@3463: // this humongous region and then set the bit range. tonyp@3957: BitMap::idx_t end_index = (BitMap::idx_t) hr->last_hc_index(); tonyp@3713: _region_bm->par_at_put_range(index, end_index, true); tonyp@1264: } tonyp@1264: } tonyp@1264: johnc@3731: public: johnc@4123: CMCountDataClosureBase(G1CollectedHeap* g1h, johnc@3731: BitMap* region_bm, BitMap* card_bm): johnc@4123: _g1h(g1h), _cm(g1h->concurrent_mark()), johnc@4123: _ct_bs((CardTableModRefBS*) (g1h->barrier_set())), johnc@4123: _region_bm(region_bm), _card_bm(card_bm) { } johnc@3731: }; johnc@3731: johnc@3731: // Closure that calculates the # live objects per region. Used johnc@3731: // for verification purposes during the cleanup pause. johnc@3731: class CalcLiveObjectsClosure: public CMCountDataClosureBase { johnc@3731: CMBitMapRO* _bm; johnc@3731: size_t _region_marked_bytes; johnc@3731: johnc@3731: public: johnc@4123: CalcLiveObjectsClosure(CMBitMapRO *bm, G1CollectedHeap* g1h, johnc@3731: BitMap* region_bm, BitMap* card_bm) : johnc@4123: CMCountDataClosureBase(g1h, region_bm, card_bm), johnc@3731: _bm(bm), _region_marked_bytes(0) { } johnc@3731: ysr@777: bool doHeapRegion(HeapRegion* hr) { ysr@777: iveresov@1074: if (hr->continuesHumongous()) { tonyp@1264: // We will ignore these here and process them when their tonyp@1264: // associated "starts humongous" region is processed (see tonyp@1264: // set_bit_for_heap_region()). Note that we cannot rely on their tonyp@1264: // associated "starts humongous" region to have their bit set to tonyp@1264: // 1 since, due to the region chunking in the parallel region tonyp@1264: // iteration, a "continues humongous" region might be visited tonyp@1264: // before its associated "starts humongous". iveresov@1074: return false; iveresov@1074: } ysr@777: johnc@4123: HeapWord* ntams = hr->next_top_at_mark_start(); johnc@4123: HeapWord* start = hr->bottom(); johnc@4123: johnc@4123: assert(start <= hr->end() && start <= ntams && ntams <= hr->end(), johnc@3463: err_msg("Preconditions not met - " johnc@4123: "start: "PTR_FORMAT", ntams: "PTR_FORMAT", end: "PTR_FORMAT, johnc@4123: start, ntams, hr->end())); johnc@3463: ysr@777: // Find the first marked object at or after "start". johnc@4123: start = _bm->getNextMarkedWordAddress(start, ntams); johnc@3463: ysr@777: size_t marked_bytes = 0; ysr@777: johnc@4123: while (start < ntams) { ysr@777: oop obj = oop(start); ysr@777: int obj_sz = obj->size(); johnc@4123: HeapWord* obj_end = start + obj_sz; johnc@3731: johnc@3731: BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start); johnc@4123: BitMap::idx_t end_idx = _cm->card_bitmap_index_for(obj_end); johnc@4123: johnc@4123: // Note: if we're looking at the last region in heap - obj_end johnc@4123: // could be actually just beyond the end of the heap; end_idx johnc@4123: // will then correspond to a (non-existent) card that is also johnc@4123: // just beyond the heap. johnc@4123: if (_g1h->is_in_g1_reserved(obj_end) && !_ct_bs->is_card_aligned(obj_end)) { johnc@4123: // end of object is not card aligned - increment to cover johnc@4123: // all the cards spanned by the object johnc@4123: end_idx += 1; johnc@4123: } johnc@4123: johnc@4123: // Set the bits in the card BM for the cards spanned by this object. johnc@4123: _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */); johnc@3731: johnc@3731: // Add the size of this object to the number of marked bytes. apetrusenko@1465: marked_bytes += (size_t)obj_sz * HeapWordSize; johnc@3463: ysr@777: // Find the next marked object after this one. johnc@4123: start = _bm->getNextMarkedWordAddress(obj_end, ntams); tonyp@2973: } johnc@3463: johnc@3463: // Mark the allocated-since-marking portion... johnc@3463: HeapWord* top = hr->top(); johnc@4123: if (ntams < top) { johnc@4123: BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams); johnc@4123: BitMap::idx_t end_idx = _cm->card_bitmap_index_for(top); johnc@4123: johnc@4123: // Note: if we're looking at the last region in heap - top johnc@4123: // could be actually just beyond the end of the heap; end_idx johnc@4123: // will then correspond to a (non-existent) card that is also johnc@4123: // just beyond the heap. johnc@4123: if (_g1h->is_in_g1_reserved(top) && !_ct_bs->is_card_aligned(top)) { johnc@4123: // end of object is not card aligned - increment to cover johnc@4123: // all the cards spanned by the object johnc@4123: end_idx += 1; johnc@4123: } johnc@4123: _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */); johnc@3463: johnc@3463: // This definitely means the region has live objects. johnc@3463: set_bit_for_region(hr); ysr@777: } ysr@777: ysr@777: // Update the live region bitmap. ysr@777: if (marked_bytes > 0) { tonyp@1264: set_bit_for_region(hr); ysr@777: } johnc@3463: johnc@3463: // Set the marked bytes for the current region so that johnc@3463: // it can be queried by a calling verificiation routine johnc@3463: _region_marked_bytes = marked_bytes; johnc@3463: johnc@3463: return false; johnc@3463: } johnc@3463: johnc@3463: size_t region_marked_bytes() const { return _region_marked_bytes; } johnc@3463: }; johnc@3463: johnc@3463: // Heap region closure used for verifying the counting data johnc@3463: // that was accumulated concurrently and aggregated during johnc@3463: // the remark pause. This closure is applied to the heap johnc@3463: // regions during the STW cleanup pause. johnc@3463: johnc@3463: class VerifyLiveObjectDataHRClosure: public HeapRegionClosure { johnc@4123: G1CollectedHeap* _g1h; johnc@3463: ConcurrentMark* _cm; johnc@3463: CalcLiveObjectsClosure _calc_cl; johnc@3463: BitMap* _region_bm; // Region BM to be verified johnc@3463: BitMap* _card_bm; // Card BM to be verified johnc@3463: bool _verbose; // verbose output? johnc@3463: johnc@3463: BitMap* _exp_region_bm; // Expected Region BM values johnc@3463: BitMap* _exp_card_bm; // Expected card BM values johnc@3463: johnc@3463: int _failures; johnc@3463: johnc@3463: public: johnc@4123: VerifyLiveObjectDataHRClosure(G1CollectedHeap* g1h, johnc@3463: BitMap* region_bm, johnc@3463: BitMap* card_bm, johnc@3463: BitMap* exp_region_bm, johnc@3463: BitMap* exp_card_bm, johnc@3463: bool verbose) : johnc@4123: _g1h(g1h), _cm(g1h->concurrent_mark()), johnc@4123: _calc_cl(_cm->nextMarkBitMap(), g1h, exp_region_bm, exp_card_bm), johnc@3463: _region_bm(region_bm), _card_bm(card_bm), _verbose(verbose), johnc@3463: _exp_region_bm(exp_region_bm), _exp_card_bm(exp_card_bm), johnc@3463: _failures(0) { } johnc@3463: johnc@3463: int failures() const { return _failures; } johnc@3463: johnc@3463: bool doHeapRegion(HeapRegion* hr) { johnc@3463: if (hr->continuesHumongous()) { johnc@3463: // We will ignore these here and process them when their johnc@3463: // associated "starts humongous" region is processed (see johnc@3463: // set_bit_for_heap_region()). Note that we cannot rely on their johnc@3463: // associated "starts humongous" region to have their bit set to johnc@3463: // 1 since, due to the region chunking in the parallel region johnc@3463: // iteration, a "continues humongous" region might be visited johnc@3463: // before its associated "starts humongous". johnc@3463: return false; johnc@3463: } johnc@3463: johnc@3463: int failures = 0; johnc@3463: johnc@3463: // Call the CalcLiveObjectsClosure to walk the marking bitmap for johnc@3463: // this region and set the corresponding bits in the expected region johnc@3463: // and card bitmaps. johnc@3463: bool res = _calc_cl.doHeapRegion(hr); johnc@3463: assert(res == false, "should be continuing"); johnc@3463: johnc@3463: MutexLockerEx x((_verbose ? ParGCRareEvent_lock : NULL), johnc@3463: Mutex::_no_safepoint_check_flag); johnc@3463: johnc@3463: // Verify the marked bytes for this region. johnc@3463: size_t exp_marked_bytes = _calc_cl.region_marked_bytes(); johnc@3463: size_t act_marked_bytes = hr->next_marked_bytes(); johnc@3463: johnc@3463: // We're not OK if expected marked bytes > actual marked bytes. It means johnc@3463: // we have missed accounting some objects during the actual marking. johnc@3463: if (exp_marked_bytes > act_marked_bytes) { johnc@3463: if (_verbose) { tonyp@3713: gclog_or_tty->print_cr("Region %u: marked bytes mismatch: " johnc@3463: "expected: " SIZE_FORMAT ", actual: " SIZE_FORMAT, johnc@3463: hr->hrs_index(), exp_marked_bytes, act_marked_bytes); johnc@3463: } johnc@3463: failures += 1; johnc@3463: } johnc@3463: johnc@3463: // Verify the bit, for this region, in the actual and expected johnc@3463: // (which was just calculated) region bit maps. johnc@3463: // We're not OK if the bit in the calculated expected region johnc@3463: // bitmap is set and the bit in the actual region bitmap is not. tonyp@3713: BitMap::idx_t index = (BitMap::idx_t) hr->hrs_index(); johnc@3463: johnc@3463: bool expected = _exp_region_bm->at(index); johnc@3463: bool actual = _region_bm->at(index); johnc@3463: if (expected && !actual) { johnc@3463: if (_verbose) { tonyp@3713: gclog_or_tty->print_cr("Region %u: region bitmap mismatch: " tonyp@3713: "expected: %s, actual: %s", tonyp@3713: hr->hrs_index(), tonyp@3713: BOOL_TO_STR(expected), BOOL_TO_STR(actual)); johnc@3463: } johnc@3463: failures += 1; johnc@3463: } johnc@3463: johnc@3463: // Verify that the card bit maps for the cards spanned by the current johnc@3463: // region match. We have an error if we have a set bit in the expected johnc@3463: // bit map and the corresponding bit in the actual bitmap is not set. johnc@3463: johnc@3463: BitMap::idx_t start_idx = _cm->card_bitmap_index_for(hr->bottom()); johnc@3463: BitMap::idx_t end_idx = _cm->card_bitmap_index_for(hr->top()); johnc@3463: johnc@3463: for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) { johnc@3463: expected = _exp_card_bm->at(i); johnc@3463: actual = _card_bm->at(i); johnc@3463: johnc@3463: if (expected && !actual) { johnc@3463: if (_verbose) { tonyp@3713: gclog_or_tty->print_cr("Region %u: card bitmap mismatch at " SIZE_FORMAT ": " tonyp@3713: "expected: %s, actual: %s", tonyp@3713: hr->hrs_index(), i, tonyp@3713: BOOL_TO_STR(expected), BOOL_TO_STR(actual)); ysr@777: } johnc@3463: failures += 1; ysr@777: } ysr@777: } ysr@777: johnc@3463: if (failures > 0 && _verbose) { johnc@3463: gclog_or_tty->print_cr("Region " HR_FORMAT ", ntams: " PTR_FORMAT ", " johnc@3463: "marked_bytes: calc/actual " SIZE_FORMAT "/" SIZE_FORMAT, johnc@3463: HR_FORMAT_PARAMS(hr), hr->next_top_at_mark_start(), johnc@3463: _calc_cl.region_marked_bytes(), hr->next_marked_bytes()); johnc@3463: } johnc@3463: johnc@3463: _failures += failures; johnc@3463: johnc@3463: // We could stop iteration over the heap when we johnc@3731: // find the first violating region by returning true. ysr@777: return false; ysr@777: } ysr@777: }; ysr@777: ysr@777: johnc@3463: class G1ParVerifyFinalCountTask: public AbstractGangTask { johnc@3463: protected: johnc@3463: G1CollectedHeap* _g1h; johnc@3463: ConcurrentMark* _cm; johnc@3463: BitMap* _actual_region_bm; johnc@3463: BitMap* _actual_card_bm; johnc@3463: johnc@3463: uint _n_workers; johnc@3463: johnc@3463: BitMap* _expected_region_bm; johnc@3463: BitMap* _expected_card_bm; johnc@3463: johnc@3463: int _failures; johnc@3463: bool _verbose; johnc@3463: johnc@3463: public: johnc@3463: G1ParVerifyFinalCountTask(G1CollectedHeap* g1h, johnc@3463: BitMap* region_bm, BitMap* card_bm, johnc@3463: BitMap* expected_region_bm, BitMap* expected_card_bm) johnc@3463: : AbstractGangTask("G1 verify final counting"), johnc@3463: _g1h(g1h), _cm(_g1h->concurrent_mark()), johnc@3463: _actual_region_bm(region_bm), _actual_card_bm(card_bm), johnc@3463: _expected_region_bm(expected_region_bm), _expected_card_bm(expected_card_bm), johnc@3463: _failures(0), _verbose(false), johnc@3463: _n_workers(0) { johnc@3463: assert(VerifyDuringGC, "don't call this otherwise"); johnc@3463: johnc@3463: // Use the value already set as the number of active threads johnc@3463: // in the call to run_task(). johnc@3463: if (G1CollectedHeap::use_parallel_gc_threads()) { johnc@3463: assert( _g1h->workers()->active_workers() > 0, johnc@3463: "Should have been previously set"); johnc@3463: _n_workers = _g1h->workers()->active_workers(); johnc@3463: } else { johnc@3463: _n_workers = 1; johnc@3463: } johnc@3463: johnc@3463: assert(_expected_card_bm->size() == _actual_card_bm->size(), "sanity"); johnc@3463: assert(_expected_region_bm->size() == _actual_region_bm->size(), "sanity"); johnc@3463: johnc@3463: _verbose = _cm->verbose_medium(); johnc@3463: } johnc@3463: johnc@3463: void work(uint worker_id) { johnc@3463: assert(worker_id < _n_workers, "invariant"); johnc@3463: johnc@4123: VerifyLiveObjectDataHRClosure verify_cl(_g1h, johnc@3463: _actual_region_bm, _actual_card_bm, johnc@3463: _expected_region_bm, johnc@3463: _expected_card_bm, johnc@3463: _verbose); johnc@3463: johnc@3463: if (G1CollectedHeap::use_parallel_gc_threads()) { johnc@3463: _g1h->heap_region_par_iterate_chunked(&verify_cl, johnc@3463: worker_id, johnc@3463: _n_workers, johnc@3463: HeapRegion::VerifyCountClaimValue); johnc@3463: } else { johnc@3463: _g1h->heap_region_iterate(&verify_cl); johnc@3463: } johnc@3463: johnc@3463: Atomic::add(verify_cl.failures(), &_failures); johnc@3463: } johnc@3463: johnc@3463: int failures() const { return _failures; } johnc@3463: }; johnc@3463: johnc@3731: // Closure that finalizes the liveness counting data. johnc@3731: // Used during the cleanup pause. johnc@3731: // Sets the bits corresponding to the interval [NTAMS, top] johnc@3731: // (which contains the implicitly live objects) in the johnc@3731: // card liveness bitmap. Also sets the bit for each region, johnc@3731: // containing live data, in the region liveness bitmap. johnc@3731: johnc@3731: class FinalCountDataUpdateClosure: public CMCountDataClosureBase { johnc@3463: public: johnc@4123: FinalCountDataUpdateClosure(G1CollectedHeap* g1h, johnc@3463: BitMap* region_bm, johnc@3463: BitMap* card_bm) : johnc@4123: CMCountDataClosureBase(g1h, region_bm, card_bm) { } johnc@3463: johnc@3463: bool doHeapRegion(HeapRegion* hr) { johnc@3463: johnc@3463: if (hr->continuesHumongous()) { johnc@3463: // We will ignore these here and process them when their johnc@3463: // associated "starts humongous" region is processed (see johnc@3463: // set_bit_for_heap_region()). Note that we cannot rely on their johnc@3463: // associated "starts humongous" region to have their bit set to johnc@3463: // 1 since, due to the region chunking in the parallel region johnc@3463: // iteration, a "continues humongous" region might be visited johnc@3463: // before its associated "starts humongous". johnc@3463: return false; johnc@3463: } johnc@3463: johnc@3463: HeapWord* ntams = hr->next_top_at_mark_start(); johnc@3463: HeapWord* top = hr->top(); johnc@3463: johnc@3731: assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions."); johnc@3463: johnc@3463: // Mark the allocated-since-marking portion... johnc@3463: if (ntams < top) { johnc@3463: // This definitely means the region has live objects. johnc@3463: set_bit_for_region(hr); johnc@4123: johnc@4123: // Now set the bits in the card bitmap for [ntams, top) johnc@4123: BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams); johnc@4123: BitMap::idx_t end_idx = _cm->card_bitmap_index_for(top); johnc@4123: johnc@4123: // Note: if we're looking at the last region in heap - top johnc@4123: // could be actually just beyond the end of the heap; end_idx johnc@4123: // will then correspond to a (non-existent) card that is also johnc@4123: // just beyond the heap. johnc@4123: if (_g1h->is_in_g1_reserved(top) && !_ct_bs->is_card_aligned(top)) { johnc@4123: // end of object is not card aligned - increment to cover johnc@4123: // all the cards spanned by the object johnc@4123: end_idx += 1; johnc@4123: } johnc@4123: johnc@4123: assert(end_idx <= _card_bm->size(), johnc@4123: err_msg("oob: end_idx= "SIZE_FORMAT", bitmap size= "SIZE_FORMAT, johnc@4123: end_idx, _card_bm->size())); johnc@4123: assert(start_idx < _card_bm->size(), johnc@4123: err_msg("oob: start_idx= "SIZE_FORMAT", bitmap size= "SIZE_FORMAT, johnc@4123: start_idx, _card_bm->size())); johnc@4123: johnc@4123: _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */); coleenp@4037: } johnc@3463: johnc@3463: // Set the bit for the region if it contains live data johnc@3463: if (hr->next_marked_bytes() > 0) { johnc@3463: set_bit_for_region(hr); johnc@3463: } johnc@3463: johnc@3463: return false; johnc@3463: } johnc@3463: }; ysr@777: ysr@777: class G1ParFinalCountTask: public AbstractGangTask { ysr@777: protected: ysr@777: G1CollectedHeap* _g1h; johnc@3463: ConcurrentMark* _cm; johnc@3463: BitMap* _actual_region_bm; johnc@3463: BitMap* _actual_card_bm; johnc@3463: jmasa@3357: uint _n_workers; johnc@3463: ysr@777: public: johnc@3463: G1ParFinalCountTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm) johnc@3463: : AbstractGangTask("G1 final counting"), johnc@3463: _g1h(g1h), _cm(_g1h->concurrent_mark()), johnc@3463: _actual_region_bm(region_bm), _actual_card_bm(card_bm), johnc@3463: _n_workers(0) { jmasa@3294: // Use the value already set as the number of active threads tonyp@3714: // in the call to run_task(). jmasa@3294: if (G1CollectedHeap::use_parallel_gc_threads()) { jmasa@3294: assert( _g1h->workers()->active_workers() > 0, jmasa@3294: "Should have been previously set"); jmasa@3294: _n_workers = _g1h->workers()->active_workers(); tonyp@2973: } else { ysr@777: _n_workers = 1; tonyp@2973: } ysr@777: } ysr@777: jmasa@3357: void work(uint worker_id) { johnc@3463: assert(worker_id < _n_workers, "invariant"); johnc@3463: johnc@4123: FinalCountDataUpdateClosure final_update_cl(_g1h, johnc@3463: _actual_region_bm, johnc@3463: _actual_card_bm); johnc@3463: jmasa@2188: if (G1CollectedHeap::use_parallel_gc_threads()) { johnc@3463: _g1h->heap_region_par_iterate_chunked(&final_update_cl, johnc@3463: worker_id, johnc@3463: _n_workers, tonyp@790: HeapRegion::FinalCountClaimValue); ysr@777: } else { johnc@3463: _g1h->heap_region_iterate(&final_update_cl); ysr@777: } ysr@777: } ysr@777: }; ysr@777: ysr@777: class G1ParNoteEndTask; ysr@777: ysr@777: class G1NoteEndOfConcMarkClosure : public HeapRegionClosure { ysr@777: G1CollectedHeap* _g1; ysr@777: int _worker_num; ysr@777: size_t _max_live_bytes; tonyp@3713: uint _regions_claimed; ysr@777: size_t _freed_bytes; tonyp@2493: FreeRegionList* _local_cleanup_list; tonyp@3268: OldRegionSet* _old_proxy_set; tonyp@2493: HumongousRegionSet* _humongous_proxy_set; tonyp@2493: HRRSCleanupTask* _hrrs_cleanup_task; ysr@777: double _claimed_region_time; ysr@777: double _max_region_time; ysr@777: ysr@777: public: ysr@777: G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, tonyp@2493: int worker_num, tonyp@2493: FreeRegionList* local_cleanup_list, tonyp@3268: OldRegionSet* old_proxy_set, tonyp@2493: HumongousRegionSet* humongous_proxy_set, johnc@3292: HRRSCleanupTask* hrrs_cleanup_task) : johnc@3292: _g1(g1), _worker_num(worker_num), johnc@3292: _max_live_bytes(0), _regions_claimed(0), johnc@3292: _freed_bytes(0), johnc@3292: _claimed_region_time(0.0), _max_region_time(0.0), johnc@3292: _local_cleanup_list(local_cleanup_list), johnc@3292: _old_proxy_set(old_proxy_set), johnc@3292: _humongous_proxy_set(humongous_proxy_set), johnc@3292: _hrrs_cleanup_task(hrrs_cleanup_task) { } johnc@3292: ysr@777: size_t freed_bytes() { return _freed_bytes; } ysr@777: johnc@3292: bool doHeapRegion(HeapRegion *hr) { tonyp@3957: if (hr->continuesHumongous()) { tonyp@3957: return false; tonyp@3957: } johnc@3292: // We use a claim value of zero here because all regions johnc@3292: // were claimed with value 1 in the FinalCount task. tonyp@3957: _g1->reset_gc_time_stamps(hr); tonyp@3957: double start = os::elapsedTime(); tonyp@3957: _regions_claimed++; tonyp@3957: hr->note_end_of_marking(); tonyp@3957: _max_live_bytes += hr->max_live_bytes(); tonyp@3957: _g1->free_region_if_empty(hr, tonyp@3957: &_freed_bytes, tonyp@3957: _local_cleanup_list, tonyp@3957: _old_proxy_set, tonyp@3957: _humongous_proxy_set, tonyp@3957: _hrrs_cleanup_task, tonyp@3957: true /* par */); tonyp@3957: double region_time = (os::elapsedTime() - start); tonyp@3957: _claimed_region_time += region_time; tonyp@3957: if (region_time > _max_region_time) { tonyp@3957: _max_region_time = region_time; johnc@3292: } johnc@3292: return false; johnc@3292: } ysr@777: ysr@777: size_t max_live_bytes() { return _max_live_bytes; } tonyp@3713: uint regions_claimed() { return _regions_claimed; } ysr@777: double claimed_region_time_sec() { return _claimed_region_time; } ysr@777: double max_region_time_sec() { return _max_region_time; } ysr@777: }; ysr@777: ysr@777: class G1ParNoteEndTask: public AbstractGangTask { ysr@777: friend class G1NoteEndOfConcMarkClosure; tonyp@2472: ysr@777: protected: ysr@777: G1CollectedHeap* _g1h; ysr@777: size_t _max_live_bytes; ysr@777: size_t _freed_bytes; tonyp@2472: FreeRegionList* _cleanup_list; tonyp@2472: ysr@777: public: ysr@777: G1ParNoteEndTask(G1CollectedHeap* g1h, tonyp@2472: FreeRegionList* cleanup_list) : ysr@777: AbstractGangTask("G1 note end"), _g1h(g1h), tonyp@2472: _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list) { } ysr@777: jmasa@3357: void work(uint worker_id) { ysr@777: double start = os::elapsedTime(); tonyp@2493: FreeRegionList local_cleanup_list("Local Cleanup List"); tonyp@3268: OldRegionSet old_proxy_set("Local Cleanup Old Proxy Set"); tonyp@2493: HumongousRegionSet humongous_proxy_set("Local Cleanup Humongous Proxy Set"); tonyp@2493: HRRSCleanupTask hrrs_cleanup_task; jmasa@3357: G1NoteEndOfConcMarkClosure g1_note_end(_g1h, worker_id, &local_cleanup_list, tonyp@3268: &old_proxy_set, tonyp@2493: &humongous_proxy_set, tonyp@2493: &hrrs_cleanup_task); jmasa@2188: if (G1CollectedHeap::use_parallel_gc_threads()) { jmasa@3357: _g1h->heap_region_par_iterate_chunked(&g1_note_end, worker_id, jmasa@3294: _g1h->workers()->active_workers(), tonyp@790: HeapRegion::NoteEndClaimValue); ysr@777: } else { ysr@777: _g1h->heap_region_iterate(&g1_note_end); ysr@777: } ysr@777: assert(g1_note_end.complete(), "Shouldn't have yielded!"); ysr@777: tonyp@2472: // Now update the lists tonyp@2472: _g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(), tonyp@2472: NULL /* free_list */, tonyp@3268: &old_proxy_set, tonyp@2493: &humongous_proxy_set, tonyp@2472: true /* par */); ysr@777: { ysr@777: MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); ysr@777: _max_live_bytes += g1_note_end.max_live_bytes(); ysr@777: _freed_bytes += g1_note_end.freed_bytes(); tonyp@2472: tonyp@2975: // If we iterate over the global cleanup list at the end of tonyp@2975: // cleanup to do this printing we will not guarantee to only tonyp@2975: // generate output for the newly-reclaimed regions (the list tonyp@2975: // might not be empty at the beginning of cleanup; we might tonyp@2975: // still be working on its previous contents). So we do the tonyp@2975: // printing here, before we append the new regions to the global tonyp@2975: // cleanup list. tonyp@2975: tonyp@2975: G1HRPrinter* hr_printer = _g1h->hr_printer(); tonyp@2975: if (hr_printer->is_active()) { tonyp@2975: HeapRegionLinkedListIterator iter(&local_cleanup_list); tonyp@2975: while (iter.more_available()) { tonyp@2975: HeapRegion* hr = iter.get_next(); tonyp@2975: hr_printer->cleanup(hr); tonyp@2975: } tonyp@2975: } tonyp@2975: tonyp@2493: _cleanup_list->add_as_tail(&local_cleanup_list); tonyp@2493: assert(local_cleanup_list.is_empty(), "post-condition"); tonyp@2493: tonyp@2493: HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task); ysr@777: } ysr@777: } ysr@777: size_t max_live_bytes() { return _max_live_bytes; } ysr@777: size_t freed_bytes() { return _freed_bytes; } ysr@777: }; ysr@777: ysr@777: class G1ParScrubRemSetTask: public AbstractGangTask { ysr@777: protected: ysr@777: G1RemSet* _g1rs; ysr@777: BitMap* _region_bm; ysr@777: BitMap* _card_bm; ysr@777: public: ysr@777: G1ParScrubRemSetTask(G1CollectedHeap* g1h, ysr@777: BitMap* region_bm, BitMap* card_bm) : ysr@777: AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()), johnc@3463: _region_bm(region_bm), _card_bm(card_bm) { } ysr@777: jmasa@3357: void work(uint worker_id) { jmasa@2188: if (G1CollectedHeap::use_parallel_gc_threads()) { jmasa@3357: _g1rs->scrub_par(_region_bm, _card_bm, worker_id, tonyp@790: HeapRegion::ScrubRemSetClaimValue); ysr@777: } else { ysr@777: _g1rs->scrub(_region_bm, _card_bm); ysr@777: } ysr@777: } ysr@777: ysr@777: }; ysr@777: ysr@777: void ConcurrentMark::cleanup() { ysr@777: // world is stopped at this checkpoint ysr@777: assert(SafepointSynchronize::is_at_safepoint(), ysr@777: "world should be stopped"); ysr@777: G1CollectedHeap* g1h = G1CollectedHeap::heap(); ysr@777: ysr@777: // If a full collection has happened, we shouldn't do this. ysr@777: if (has_aborted()) { ysr@777: g1h->set_marking_complete(); // So bitmap clearing isn't confused ysr@777: return; ysr@777: } ysr@777: tonyp@3268: HRSPhaseSetter x(HRSPhaseCleanup); tonyp@2472: g1h->verify_region_sets_optional(); tonyp@2472: ysr@1280: if (VerifyDuringGC) { ysr@1280: HandleMark hm; // handle scope ysr@1280: Universe::heap()->prepare_for_verify(); stefank@5018: Universe::verify(VerifyOption_G1UsePrevMarking, stefank@5018: " VerifyDuringGC:(before)"); ysr@1280: } ysr@1280: ysr@777: G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy(); ysr@777: g1p->record_concurrent_mark_cleanup_start(); ysr@777: ysr@777: double start = os::elapsedTime(); ysr@777: tonyp@2493: HeapRegionRemSet::reset_for_cleanup_tasks(); tonyp@2493: jmasa@3357: uint n_workers; jmasa@3294: ysr@777: // Do counting once more with the world stopped for good measure. johnc@3463: G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm); johnc@3463: jmasa@2188: if (G1CollectedHeap::use_parallel_gc_threads()) { johnc@3463: assert(g1h->check_heap_region_claim_values(HeapRegion::InitialClaimValue), tonyp@790: "sanity check"); tonyp@790: johnc@3338: g1h->set_par_threads(); johnc@3338: n_workers = g1h->n_par_threads(); jmasa@3357: assert(g1h->n_par_threads() == n_workers, johnc@3338: "Should not have been reset"); ysr@777: g1h->workers()->run_task(&g1_par_count_task); jmasa@3294: // Done with the parallel phase so reset to 0. ysr@777: g1h->set_par_threads(0); tonyp@790: johnc@3463: assert(g1h->check_heap_region_claim_values(HeapRegion::FinalCountClaimValue), tonyp@790: "sanity check"); ysr@777: } else { johnc@3338: n_workers = 1; ysr@777: g1_par_count_task.work(0); ysr@777: } ysr@777: johnc@3463: if (VerifyDuringGC) { johnc@3463: // Verify that the counting data accumulated during marking matches johnc@3463: // that calculated by walking the marking bitmap. johnc@3463: johnc@3463: // Bitmaps to hold expected values johnc@3463: BitMap expected_region_bm(_region_bm.size(), false); johnc@3463: BitMap expected_card_bm(_card_bm.size(), false); johnc@3463: johnc@3463: G1ParVerifyFinalCountTask g1_par_verify_task(g1h, johnc@3463: &_region_bm, johnc@3463: &_card_bm, johnc@3463: &expected_region_bm, johnc@3463: &expected_card_bm); johnc@3463: johnc@3463: if (G1CollectedHeap::use_parallel_gc_threads()) { johnc@3463: g1h->set_par_threads((int)n_workers); johnc@3463: g1h->workers()->run_task(&g1_par_verify_task); johnc@3463: // Done with the parallel phase so reset to 0. johnc@3463: g1h->set_par_threads(0); johnc@3463: johnc@3463: assert(g1h->check_heap_region_claim_values(HeapRegion::VerifyCountClaimValue), johnc@3463: "sanity check"); johnc@3463: } else { johnc@3463: g1_par_verify_task.work(0); johnc@3463: } johnc@3463: johnc@3463: guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures"); johnc@3463: } johnc@3463: ysr@777: size_t start_used_bytes = g1h->used(); ysr@777: g1h->set_marking_complete(); ysr@777: ysr@777: double count_end = os::elapsedTime(); ysr@777: double this_final_counting_time = (count_end - start); ysr@777: _total_counting_time += this_final_counting_time; ysr@777: tonyp@2717: if (G1PrintRegionLivenessInfo) { tonyp@2717: G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking"); tonyp@2717: _g1h->heap_region_iterate(&cl); tonyp@2717: } tonyp@2717: ysr@777: // Install newly created mark bitMap as "prev". ysr@777: swapMarkBitMaps(); ysr@777: ysr@777: g1h->reset_gc_time_stamp(); ysr@777: ysr@777: // Note end of marking in all heap regions. tonyp@2472: G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list); jmasa@2188: if (G1CollectedHeap::use_parallel_gc_threads()) { jmasa@3294: g1h->set_par_threads((int)n_workers); ysr@777: g1h->workers()->run_task(&g1_par_note_end_task); ysr@777: g1h->set_par_threads(0); tonyp@790: tonyp@790: assert(g1h->check_heap_region_claim_values(HeapRegion::NoteEndClaimValue), tonyp@790: "sanity check"); ysr@777: } else { ysr@777: g1_par_note_end_task.work(0); ysr@777: } tonyp@3957: g1h->check_gc_time_stamps(); tonyp@2472: tonyp@2472: if (!cleanup_list_is_empty()) { tonyp@2472: // The cleanup list is not empty, so we'll have to process it tonyp@2472: // concurrently. Notify anyone else that might be wanting free tonyp@2472: // regions that there will be more free regions coming soon. tonyp@2472: g1h->set_free_regions_coming(); tonyp@2472: } ysr@777: ysr@777: // call below, since it affects the metric by which we sort the heap ysr@777: // regions. ysr@777: if (G1ScrubRemSets) { ysr@777: double rs_scrub_start = os::elapsedTime(); ysr@777: G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm); jmasa@2188: if (G1CollectedHeap::use_parallel_gc_threads()) { jmasa@3294: g1h->set_par_threads((int)n_workers); ysr@777: g1h->workers()->run_task(&g1_par_scrub_rs_task); ysr@777: g1h->set_par_threads(0); tonyp@790: tonyp@790: assert(g1h->check_heap_region_claim_values( tonyp@790: HeapRegion::ScrubRemSetClaimValue), tonyp@790: "sanity check"); ysr@777: } else { ysr@777: g1_par_scrub_rs_task.work(0); ysr@777: } ysr@777: ysr@777: double rs_scrub_end = os::elapsedTime(); ysr@777: double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start); ysr@777: _total_rs_scrub_time += this_rs_scrub_time; ysr@777: } ysr@777: ysr@777: // this will also free any regions totally full of garbage objects, ysr@777: // and sort the regions. jmasa@3294: g1h->g1_policy()->record_concurrent_mark_cleanup_end((int)n_workers); ysr@777: ysr@777: // Statistics. ysr@777: double end = os::elapsedTime(); ysr@777: _cleanup_times.add((end - start) * 1000.0); ysr@777: brutisso@3710: if (G1Log::fine()) { ysr@777: g1h->print_size_transition(gclog_or_tty, ysr@777: start_used_bytes, ysr@777: g1h->used(), ysr@777: g1h->capacity()); ysr@777: } ysr@777: johnc@3175: // Clean up will have freed any regions completely full of garbage. johnc@3175: // Update the soft reference policy with the new heap occupancy. johnc@3175: Universe::update_heap_info_at_gc(); johnc@3175: ysr@777: // We need to make this be a "collection" so any collection pause that ysr@777: // races with it goes around and waits for completeCleanup to finish. ysr@777: g1h->increment_total_collections(); ysr@777: tonyp@3457: // We reclaimed old regions so we should calculate the sizes to make tonyp@3457: // sure we update the old gen/space data. tonyp@3457: g1h->g1mm()->update_sizes(); tonyp@3457: johnc@1186: if (VerifyDuringGC) { ysr@1280: HandleMark hm; // handle scope ysr@1280: Universe::heap()->prepare_for_verify(); stefank@5018: Universe::verify(VerifyOption_G1UsePrevMarking, stefank@5018: " VerifyDuringGC:(after)"); ysr@777: } tonyp@2472: tonyp@2472: g1h->verify_region_sets_optional(); sla@5237: g1h->trace_heap_after_concurrent_cycle(); ysr@777: } ysr@777: ysr@777: void ConcurrentMark::completeCleanup() { ysr@777: if (has_aborted()) return; ysr@777: tonyp@2472: G1CollectedHeap* g1h = G1CollectedHeap::heap(); tonyp@2472: tonyp@2472: _cleanup_list.verify_optional(); tonyp@2643: FreeRegionList tmp_free_list("Tmp Free List"); tonyp@2472: tonyp@2472: if (G1ConcRegionFreeingVerbose) { tonyp@2472: gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " tonyp@3713: "cleanup list has %u entries", tonyp@2472: _cleanup_list.length()); tonyp@2472: } tonyp@2472: tonyp@2472: // Noone else should be accessing the _cleanup_list at this point, tonyp@2472: // so it's not necessary to take any locks tonyp@2472: while (!_cleanup_list.is_empty()) { tonyp@2472: HeapRegion* hr = _cleanup_list.remove_head(); tonyp@2472: assert(hr != NULL, "the list was not empty"); tonyp@2849: hr->par_clear(); tonyp@2643: tmp_free_list.add_as_tail(hr); tonyp@2472: tonyp@2472: // Instead of adding one region at a time to the secondary_free_list, tonyp@2472: // we accumulate them in the local list and move them a few at a tonyp@2472: // time. This also cuts down on the number of notify_all() calls tonyp@2472: // we do during this process. We'll also append the local list when tonyp@2472: // _cleanup_list is empty (which means we just removed the last tonyp@2472: // region from the _cleanup_list). tonyp@2643: if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) || tonyp@2472: _cleanup_list.is_empty()) { tonyp@2472: if (G1ConcRegionFreeingVerbose) { tonyp@2472: gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " tonyp@3713: "appending %u entries to the secondary_free_list, " tonyp@3713: "cleanup list still has %u entries", tonyp@2643: tmp_free_list.length(), tonyp@2472: _cleanup_list.length()); ysr@777: } tonyp@2472: tonyp@2472: { tonyp@2472: MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); tonyp@2643: g1h->secondary_free_list_add_as_tail(&tmp_free_list); tonyp@2472: SecondaryFreeList_lock->notify_all(); tonyp@2472: } tonyp@2472: tonyp@2472: if (G1StressConcRegionFreeing) { tonyp@2472: for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) { tonyp@2472: os::sleep(Thread::current(), (jlong) 1, false); tonyp@2472: } tonyp@2472: } ysr@777: } ysr@777: } tonyp@2643: assert(tmp_free_list.is_empty(), "post-condition"); ysr@777: } ysr@777: johnc@4555: // Supporting Object and Oop closures for reference discovery johnc@4555: // and processing in during marking johnc@2494: johnc@2379: bool G1CMIsAliveClosure::do_object_b(oop obj) { johnc@2379: HeapWord* addr = (HeapWord*)obj; johnc@2379: return addr != NULL && johnc@2379: (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj)); johnc@2379: } ysr@777: johnc@4555: // 'Keep Alive' oop closure used by both serial parallel reference processing. johnc@4555: // Uses the CMTask associated with a worker thread (for serial reference johnc@4555: // processing the CMTask for worker 0 is used) to preserve (mark) and johnc@4555: // trace referent objects. johnc@4555: // johnc@4555: // Using the CMTask and embedded local queues avoids having the worker johnc@4555: // threads operating on the global mark stack. This reduces the risk johnc@4555: // of overflowing the stack - which we would rather avoid at this late johnc@4555: // state. Also using the tasks' local queues removes the potential johnc@4555: // of the workers interfering with each other that could occur if johnc@4555: // operating on the global stack. johnc@4555: johnc@4555: class G1CMKeepAliveAndDrainClosure: public OopClosure { johnc@4787: ConcurrentMark* _cm; johnc@4787: CMTask* _task; johnc@4787: int _ref_counter_limit; johnc@4787: int _ref_counter; johnc@4787: bool _is_serial; johnc@2494: public: johnc@4787: G1CMKeepAliveAndDrainClosure(ConcurrentMark* cm, CMTask* task, bool is_serial) : johnc@4787: _cm(cm), _task(task), _is_serial(is_serial), johnc@4787: _ref_counter_limit(G1RefProcDrainInterval) { johnc@2494: assert(_ref_counter_limit > 0, "sanity"); johnc@4787: assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code"); johnc@2494: _ref_counter = _ref_counter_limit; johnc@2494: } johnc@2494: johnc@2494: virtual void do_oop(narrowOop* p) { do_oop_work(p); } johnc@2494: virtual void do_oop( oop* p) { do_oop_work(p); } johnc@2494: johnc@2494: template void do_oop_work(T* p) { johnc@2494: if (!_cm->has_overflown()) { johnc@2494: oop obj = oopDesc::load_decode_heap_oop(p); tonyp@2973: if (_cm->verbose_high()) { johnc@4173: gclog_or_tty->print_cr("\t[%u] we're looking at location " johnc@2494: "*"PTR_FORMAT" = "PTR_FORMAT, johnc@4173: _task->worker_id(), p, (void*) obj); tonyp@2973: } johnc@2494: johnc@2494: _task->deal_with_reference(obj); johnc@2494: _ref_counter--; johnc@2494: johnc@2494: if (_ref_counter == 0) { johnc@4555: // We have dealt with _ref_counter_limit references, pushing them johnc@4555: // and objects reachable from them on to the local stack (and johnc@4555: // possibly the global stack). Call CMTask::do_marking_step() to johnc@4555: // process these entries. johnc@4555: // johnc@4555: // We call CMTask::do_marking_step() in a loop, which we'll exit if johnc@4555: // there's nothing more to do (i.e. we're done with the entries that johnc@4555: // were pushed as a result of the CMTask::deal_with_reference() calls johnc@4555: // above) or we overflow. johnc@4555: // johnc@4555: // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() johnc@4555: // flag while there may still be some work to do. (See the comment at johnc@4555: // the beginning of CMTask::do_marking_step() for those conditions - johnc@4555: // one of which is reaching the specified time target.) It is only johnc@4555: // when CMTask::do_marking_step() returns without setting the johnc@4555: // has_aborted() flag that the marking step has completed. johnc@2494: do { johnc@2494: double mark_step_duration_ms = G1ConcMarkStepDurationMillis; johnc@2494: _task->do_marking_step(mark_step_duration_ms, johnc@4787: false /* do_termination */, johnc@4787: _is_serial); johnc@2494: } while (_task->has_aborted() && !_cm->has_overflown()); johnc@2494: _ref_counter = _ref_counter_limit; johnc@2494: } johnc@2494: } else { tonyp@2973: if (_cm->verbose_high()) { johnc@4173: gclog_or_tty->print_cr("\t[%u] CM Overflow", _task->worker_id()); tonyp@2973: } johnc@2494: } johnc@2494: } johnc@2494: }; johnc@2494: johnc@4555: // 'Drain' oop closure used by both serial and parallel reference processing. johnc@4555: // Uses the CMTask associated with a given worker thread (for serial johnc@4555: // reference processing the CMtask for worker 0 is used). Calls the johnc@4555: // do_marking_step routine, with an unbelievably large timeout value, johnc@4555: // to drain the marking data structures of the remaining entries johnc@4555: // added by the 'keep alive' oop closure above. johnc@4555: johnc@4555: class G1CMDrainMarkingStackClosure: public VoidClosure { johnc@2494: ConcurrentMark* _cm; johnc@4555: CMTask* _task; johnc@4787: bool _is_serial; johnc@2494: public: johnc@4787: G1CMDrainMarkingStackClosure(ConcurrentMark* cm, CMTask* task, bool is_serial) : johnc@4787: _cm(cm), _task(task), _is_serial(is_serial) { johnc@4787: assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code"); johnc@4555: } johnc@2494: johnc@2494: void do_void() { johnc@2494: do { tonyp@2973: if (_cm->verbose_high()) { johnc@4787: gclog_or_tty->print_cr("\t[%u] Drain: Calling do_marking_step - serial: %s", johnc@4787: _task->worker_id(), BOOL_TO_STR(_is_serial)); tonyp@2973: } johnc@2494: johnc@4555: // We call CMTask::do_marking_step() to completely drain the local johnc@4555: // and global marking stacks of entries pushed by the 'keep alive' johnc@4555: // oop closure (an instance of G1CMKeepAliveAndDrainClosure above). johnc@4555: // johnc@4555: // CMTask::do_marking_step() is called in a loop, which we'll exit johnc@4555: // if there's nothing more to do (i.e. we'completely drained the johnc@4555: // entries that were pushed as a a result of applying the 'keep alive' johnc@4555: // closure to the entries on the discovered ref lists) or we overflow johnc@4555: // the global marking stack. johnc@4555: // johnc@4555: // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() johnc@4555: // flag while there may still be some work to do. (See the comment at johnc@4555: // the beginning of CMTask::do_marking_step() for those conditions - johnc@4555: // one of which is reaching the specified time target.) It is only johnc@4555: // when CMTask::do_marking_step() returns without setting the johnc@4555: // has_aborted() flag that the marking step has completed. johnc@2494: johnc@2494: _task->do_marking_step(1000000000.0 /* something very large */, johnc@4787: true /* do_termination */, johnc@4787: _is_serial); johnc@2494: } while (_task->has_aborted() && !_cm->has_overflown()); johnc@2494: } johnc@2494: }; johnc@2494: johnc@3175: // Implementation of AbstractRefProcTaskExecutor for parallel johnc@3175: // reference processing at the end of G1 concurrent marking johnc@3175: johnc@3175: class G1CMRefProcTaskExecutor: public AbstractRefProcTaskExecutor { johnc@2494: private: johnc@2494: G1CollectedHeap* _g1h; johnc@2494: ConcurrentMark* _cm; johnc@2494: WorkGang* _workers; johnc@2494: int _active_workers; johnc@2494: johnc@2494: public: johnc@3175: G1CMRefProcTaskExecutor(G1CollectedHeap* g1h, johnc@2494: ConcurrentMark* cm, johnc@2494: WorkGang* workers, johnc@2494: int n_workers) : johnc@3292: _g1h(g1h), _cm(cm), johnc@3292: _workers(workers), _active_workers(n_workers) { } johnc@2494: johnc@2494: // Executes the given task using concurrent marking worker threads. johnc@2494: virtual void execute(ProcessTask& task); johnc@2494: virtual void execute(EnqueueTask& task); johnc@2494: }; johnc@2494: johnc@3175: class G1CMRefProcTaskProxy: public AbstractGangTask { johnc@2494: typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; johnc@2494: ProcessTask& _proc_task; johnc@2494: G1CollectedHeap* _g1h; johnc@2494: ConcurrentMark* _cm; johnc@2494: johnc@2494: public: johnc@3175: G1CMRefProcTaskProxy(ProcessTask& proc_task, johnc@2494: G1CollectedHeap* g1h, johnc@3292: ConcurrentMark* cm) : johnc@2494: AbstractGangTask("Process reference objects in parallel"), johnc@4555: _proc_task(proc_task), _g1h(g1h), _cm(cm) { johnc@4787: ReferenceProcessor* rp = _g1h->ref_processor_cm(); johnc@4787: assert(rp->processing_is_mt(), "shouldn't be here otherwise"); johnc@4787: } johnc@2494: jmasa@3357: virtual void work(uint worker_id) { johnc@4787: CMTask* task = _cm->task(worker_id); johnc@2494: G1CMIsAliveClosure g1_is_alive(_g1h); johnc@4787: G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */); johnc@4787: G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */); johnc@2494: jmasa@3357: _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain); johnc@2494: } johnc@2494: }; johnc@2494: johnc@3175: void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) { johnc@2494: assert(_workers != NULL, "Need parallel worker threads."); johnc@4555: assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT"); johnc@2494: johnc@3292: G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm); johnc@2494: johnc@4788: // We need to reset the concurrency level before each johnc@4788: // proxy task execution, so that the termination protocol johnc@4788: // and overflow handling in CMTask::do_marking_step() knows johnc@4788: // how many workers to wait for. johnc@4788: _cm->set_concurrency(_active_workers); johnc@2494: _g1h->set_par_threads(_active_workers); johnc@2494: _workers->run_task(&proc_task_proxy); johnc@2494: _g1h->set_par_threads(0); johnc@2494: } johnc@2494: johnc@3175: class G1CMRefEnqueueTaskProxy: public AbstractGangTask { johnc@2494: typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; johnc@2494: EnqueueTask& _enq_task; johnc@2494: johnc@2494: public: johnc@3175: G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) : johnc@2494: AbstractGangTask("Enqueue reference objects in parallel"), johnc@3292: _enq_task(enq_task) { } johnc@2494: jmasa@3357: virtual void work(uint worker_id) { jmasa@3357: _enq_task.work(worker_id); johnc@2494: } johnc@2494: }; johnc@2494: johnc@3175: void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) { johnc@2494: assert(_workers != NULL, "Need parallel worker threads."); johnc@4555: assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT"); johnc@2494: johnc@3175: G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task); johnc@2494: johnc@4788: // Not strictly necessary but... johnc@4788: // johnc@4788: // We need to reset the concurrency level before each johnc@4788: // proxy task execution, so that the termination protocol johnc@4788: // and overflow handling in CMTask::do_marking_step() knows johnc@4788: // how many workers to wait for. johnc@4788: _cm->set_concurrency(_active_workers); johnc@2494: _g1h->set_par_threads(_active_workers); johnc@2494: _workers->run_task(&enq_task_proxy); johnc@2494: _g1h->set_par_threads(0); johnc@2494: } johnc@2494: ysr@777: void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) { johnc@4788: if (has_overflown()) { johnc@4788: // Skip processing the discovered references if we have johnc@4788: // overflown the global marking stack. Reference objects johnc@4788: // only get discovered once so it is OK to not johnc@4788: // de-populate the discovered reference lists. We could have, johnc@4788: // but the only benefit would be that, when marking restarts, johnc@4788: // less reference objects are discovered. johnc@4788: return; johnc@4788: } johnc@4788: ysr@777: ResourceMark rm; ysr@777: HandleMark hm; johnc@3171: johnc@3171: G1CollectedHeap* g1h = G1CollectedHeap::heap(); johnc@3171: johnc@3171: // Is alive closure. johnc@3171: G1CMIsAliveClosure g1_is_alive(g1h); johnc@3171: johnc@3171: // Inner scope to exclude the cleaning of the string and symbol johnc@3171: // tables from the displayed time. johnc@3171: { brutisso@3710: if (G1Log::finer()) { johnc@3171: gclog_or_tty->put(' '); johnc@3171: } sla@5237: GCTraceTime t("GC ref-proc", G1Log::finer(), false, g1h->gc_timer_cm()); johnc@3171: johnc@3175: ReferenceProcessor* rp = g1h->ref_processor_cm(); johnc@3171: johnc@3171: // See the comment in G1CollectedHeap::ref_processing_init() johnc@3171: // about how reference processing currently works in G1. johnc@3171: johnc@4555: // Set the soft reference policy johnc@3171: rp->setup_policy(clear_all_soft_refs); johnc@3171: assert(_markStack.isEmpty(), "mark stack should be empty"); johnc@3171: johnc@4787: // Instances of the 'Keep Alive' and 'Complete GC' closures used johnc@4787: // in serial reference processing. Note these closures are also johnc@4787: // used for serially processing (by the the current thread) the johnc@4787: // JNI references during parallel reference processing. johnc@4787: // johnc@4787: // These closures do not need to synchronize with the worker johnc@4787: // threads involved in parallel reference processing as these johnc@4787: // instances are executed serially by the current thread (e.g. johnc@4787: // reference processing is not multi-threaded and is thus johnc@4787: // performed by the current thread instead of a gang worker). johnc@4787: // johnc@4787: // The gang tasks involved in parallel reference procssing create johnc@4787: // their own instances of these closures, which do their own johnc@4787: // synchronization among themselves. johnc@4787: G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */); johnc@4787: G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */); johnc@4787: johnc@4787: // We need at least one active thread. If reference processing johnc@4787: // is not multi-threaded we use the current (VMThread) thread, johnc@4787: // otherwise we use the work gang from the G1CollectedHeap and johnc@4787: // we utilize all the worker threads we can. johnc@4787: bool processing_is_mt = rp->processing_is_mt() && g1h->workers() != NULL; johnc@4787: uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U); johnc@4173: active_workers = MAX2(MIN2(active_workers, _max_worker_id), 1U); johnc@3171: johnc@4787: // Parallel processing task executor. johnc@3292: G1CMRefProcTaskExecutor par_task_executor(g1h, this, johnc@3175: g1h->workers(), active_workers); johnc@4787: AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL); johnc@4555: johnc@4788: // Set the concurrency level. The phase was already set prior to johnc@4788: // executing the remark task. johnc@4788: set_concurrency(active_workers); johnc@4788: johnc@4555: // Set the degree of MT processing here. If the discovery was done MT, johnc@4555: // the number of threads involved during discovery could differ from johnc@4555: // the number of active workers. This is OK as long as the discovered johnc@4555: // Reference lists are balanced (see balance_all_queues() and balance_queues()). johnc@4555: rp->set_active_mt_degree(active_workers); johnc@4555: johnc@4555: // Process the weak references. sla@5237: const ReferenceProcessorStats& stats = sla@5237: rp->process_discovered_references(&g1_is_alive, sla@5237: &g1_keep_alive, sla@5237: &g1_drain_mark_stack, sla@5237: executor, sla@5237: g1h->gc_timer_cm()); sla@5237: g1h->gc_tracer_cm()->report_gc_reference_stats(stats); johnc@4555: johnc@4555: // The do_oop work routines of the keep_alive and drain_marking_stack johnc@4555: // oop closures will set the has_overflown flag if we overflow the johnc@4555: // global marking stack. johnc@3171: johnc@3171: assert(_markStack.overflow() || _markStack.isEmpty(), johnc@3171: "mark stack should be empty (unless it overflowed)"); johnc@4787: johnc@3171: if (_markStack.overflow()) { johnc@4555: // This should have been done already when we tried to push an johnc@3171: // entry on to the global mark stack. But let's do it again. johnc@3171: set_has_overflown(); johnc@3171: } johnc@3171: johnc@4555: assert(rp->num_q() == active_workers, "why not"); johnc@4555: johnc@4555: rp->enqueue_discovered_references(executor); johnc@3171: johnc@3171: rp->verify_no_references_recorded(); johnc@3175: assert(!rp->discovery_enabled(), "Post condition"); johnc@2494: } johnc@2494: coleenp@2497: // Now clean up stale oops in StringTable johnc@2379: StringTable::unlink(&g1_is_alive); coleenp@2497: // Clean up unreferenced symbols in symbol table. coleenp@2497: SymbolTable::unlink(); ysr@777: } ysr@777: ysr@777: void ConcurrentMark::swapMarkBitMaps() { ysr@777: CMBitMapRO* temp = _prevMarkBitMap; ysr@777: _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap; ysr@777: _nextMarkBitMap = (CMBitMap*) temp; ysr@777: } ysr@777: ysr@777: class CMRemarkTask: public AbstractGangTask { ysr@777: private: johnc@4787: ConcurrentMark* _cm; johnc@4787: bool _is_serial; ysr@777: public: jmasa@3357: void work(uint worker_id) { ysr@777: // Since all available tasks are actually started, we should ysr@777: // only proceed if we're supposed to be actived. jmasa@3357: if (worker_id < _cm->active_tasks()) { jmasa@3357: CMTask* task = _cm->task(worker_id); ysr@777: task->record_start_time(); ysr@777: do { johnc@2494: task->do_marking_step(1000000000.0 /* something very large */, johnc@4787: true /* do_termination */, johnc@4787: _is_serial); ysr@777: } while (task->has_aborted() && !_cm->has_overflown()); ysr@777: // If we overflow, then we do not want to restart. We instead ysr@777: // want to abort remark and do concurrent marking again. ysr@777: task->record_end_time(); ysr@777: } ysr@777: } ysr@777: johnc@4787: CMRemarkTask(ConcurrentMark* cm, int active_workers, bool is_serial) : johnc@4787: AbstractGangTask("Par Remark"), _cm(cm), _is_serial(is_serial) { johnc@3338: _cm->terminator()->reset_for_reuse(active_workers); jmasa@3294: } ysr@777: }; ysr@777: ysr@777: void ConcurrentMark::checkpointRootsFinalWork() { ysr@777: ResourceMark rm; ysr@777: HandleMark hm; ysr@777: G1CollectedHeap* g1h = G1CollectedHeap::heap(); ysr@777: ysr@777: g1h->ensure_parsability(false); ysr@777: jmasa@2188: if (G1CollectedHeap::use_parallel_gc_threads()) { jrose@1424: G1CollectedHeap::StrongRootsScope srs(g1h); jmasa@3294: // this is remark, so we'll use up all active threads jmasa@3357: uint active_workers = g1h->workers()->active_workers(); jmasa@3294: if (active_workers == 0) { jmasa@3294: assert(active_workers > 0, "Should have been set earlier"); jmasa@3357: active_workers = (uint) ParallelGCThreads; jmasa@3294: g1h->workers()->set_active_workers(active_workers); jmasa@3294: } johnc@4788: set_concurrency_and_phase(active_workers, false /* concurrent */); jmasa@3294: // Leave _parallel_marking_threads at it's jmasa@3294: // value originally calculated in the ConcurrentMark jmasa@3294: // constructor and pass values of the active workers jmasa@3294: // through the gang in the task. ysr@777: johnc@4787: CMRemarkTask remarkTask(this, active_workers, false /* is_serial */); johnc@4787: // We will start all available threads, even if we decide that the johnc@4787: // active_workers will be fewer. The extra ones will just bail out johnc@4787: // immediately. jmasa@3294: g1h->set_par_threads(active_workers); ysr@777: g1h->workers()->run_task(&remarkTask); ysr@777: g1h->set_par_threads(0); ysr@777: } else { jrose@1424: G1CollectedHeap::StrongRootsScope srs(g1h); jmasa@3357: uint active_workers = 1; johnc@4788: set_concurrency_and_phase(active_workers, false /* concurrent */); ysr@777: johnc@4787: // Note - if there's no work gang then the VMThread will be johnc@4787: // the thread to execute the remark - serially. We have johnc@4787: // to pass true for the is_serial parameter so that johnc@4787: // CMTask::do_marking_step() doesn't enter the sync johnc@4787: // barriers in the event of an overflow. Doing so will johnc@4787: // cause an assert that the current thread is not a johnc@4787: // concurrent GC thread. johnc@4787: CMRemarkTask remarkTask(this, active_workers, true /* is_serial*/); ysr@777: remarkTask.work(0); ysr@777: } tonyp@1458: SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); johnc@4789: guarantee(has_overflown() || johnc@4789: satb_mq_set.completed_buffers_num() == 0, johnc@4789: err_msg("Invariant: has_overflown = %s, num buffers = %d", johnc@4789: BOOL_TO_STR(has_overflown()), johnc@4789: satb_mq_set.completed_buffers_num())); ysr@777: ysr@777: print_stats(); ysr@777: } ysr@777: tonyp@1479: #ifndef PRODUCT tonyp@1479: tonyp@1823: class PrintReachableOopClosure: public OopClosure { ysr@777: private: ysr@777: G1CollectedHeap* _g1h; ysr@777: outputStream* _out; johnc@2969: VerifyOption _vo; tonyp@1823: bool _all; ysr@777: ysr@777: public: johnc@2969: PrintReachableOopClosure(outputStream* out, johnc@2969: VerifyOption vo, tonyp@1823: bool all) : tonyp@1479: _g1h(G1CollectedHeap::heap()), johnc@2969: _out(out), _vo(vo), _all(all) { } ysr@777: ysr@1280: void do_oop(narrowOop* p) { do_oop_work(p); } ysr@1280: void do_oop( oop* p) { do_oop_work(p); } ysr@1280: ysr@1280: template void do_oop_work(T* p) { ysr@1280: oop obj = oopDesc::load_decode_heap_oop(p); ysr@777: const char* str = NULL; ysr@777: const char* str2 = ""; ysr@777: tonyp@1823: if (obj == NULL) { tonyp@1823: str = ""; tonyp@1823: } else if (!_g1h->is_in_g1_reserved(obj)) { tonyp@1823: str = " O"; tonyp@1823: } else { ysr@777: HeapRegion* hr = _g1h->heap_region_containing(obj); tonyp@1458: guarantee(hr != NULL, "invariant"); tonyp@3957: bool over_tams = _g1h->allocated_since_marking(obj, hr, _vo); tonyp@3957: bool marked = _g1h->is_marked(obj, _vo); tonyp@1479: tonyp@1479: if (over_tams) { tonyp@1823: str = " >"; tonyp@1823: if (marked) { ysr@777: str2 = " AND MARKED"; tonyp@1479: } tonyp@1823: } else if (marked) { tonyp@1823: str = " M"; tonyp@1479: } else { tonyp@1823: str = " NOT"; tonyp@1479: } ysr@777: } ysr@777: tonyp@1823: _out->print_cr(" "PTR_FORMAT": "PTR_FORMAT"%s%s", ysr@777: p, (void*) obj, str, str2); ysr@777: } ysr@777: }; ysr@777: tonyp@1823: class PrintReachableObjectClosure : public ObjectClosure { ysr@777: private: johnc@2969: G1CollectedHeap* _g1h; johnc@2969: outputStream* _out; johnc@2969: VerifyOption _vo; johnc@2969: bool _all; johnc@2969: HeapRegion* _hr; ysr@777: ysr@777: public: johnc@2969: PrintReachableObjectClosure(outputStream* out, johnc@2969: VerifyOption vo, tonyp@1823: bool all, tonyp@1823: HeapRegion* hr) : johnc@2969: _g1h(G1CollectedHeap::heap()), johnc@2969: _out(out), _vo(vo), _all(all), _hr(hr) { } tonyp@1823: tonyp@1823: void do_object(oop o) { tonyp@3957: bool over_tams = _g1h->allocated_since_marking(o, _hr, _vo); tonyp@3957: bool marked = _g1h->is_marked(o, _vo); tonyp@1823: bool print_it = _all || over_tams || marked; tonyp@1823: tonyp@1823: if (print_it) { tonyp@1823: _out->print_cr(" "PTR_FORMAT"%s", hseigel@5784: (void *)o, (over_tams) ? " >" : (marked) ? " M" : ""); johnc@2969: PrintReachableOopClosure oopCl(_out, _vo, _all); coleenp@4037: o->oop_iterate_no_header(&oopCl); tonyp@1823: } ysr@777: } ysr@777: }; ysr@777: tonyp@1823: class PrintReachableRegionClosure : public HeapRegionClosure { ysr@777: private: tonyp@3957: G1CollectedHeap* _g1h; tonyp@3957: outputStream* _out; tonyp@3957: VerifyOption _vo; tonyp@3957: bool _all; ysr@777: ysr@777: public: ysr@777: bool doHeapRegion(HeapRegion* hr) { ysr@777: HeapWord* b = hr->bottom(); ysr@777: HeapWord* e = hr->end(); ysr@777: HeapWord* t = hr->top(); tonyp@3957: HeapWord* p = _g1h->top_at_mark_start(hr, _vo); ysr@777: _out->print_cr("** ["PTR_FORMAT", "PTR_FORMAT"] top: "PTR_FORMAT" " tonyp@1479: "TAMS: "PTR_FORMAT, b, e, t, p); tonyp@1823: _out->cr(); tonyp@1823: tonyp@1823: HeapWord* from = b; tonyp@1823: HeapWord* to = t; tonyp@1823: tonyp@1823: if (to > from) { tonyp@1823: _out->print_cr("Objects in ["PTR_FORMAT", "PTR_FORMAT"]", from, to); tonyp@1823: _out->cr(); johnc@2969: PrintReachableObjectClosure ocl(_out, _vo, _all, hr); tonyp@1823: hr->object_iterate_mem_careful(MemRegion(from, to), &ocl); tonyp@1823: _out->cr(); tonyp@1823: } ysr@777: ysr@777: return false; ysr@777: } ysr@777: johnc@2969: PrintReachableRegionClosure(outputStream* out, johnc@2969: VerifyOption vo, tonyp@1823: bool all) : tonyp@3957: _g1h(G1CollectedHeap::heap()), _out(out), _vo(vo), _all(all) { } ysr@777: }; ysr@777: tonyp@1823: void ConcurrentMark::print_reachable(const char* str, johnc@2969: VerifyOption vo, tonyp@1823: bool all) { tonyp@1823: gclog_or_tty->cr(); tonyp@1823: gclog_or_tty->print_cr("== Doing heap dump... "); tonyp@1479: tonyp@1479: if (G1PrintReachableBaseFile == NULL) { tonyp@1479: gclog_or_tty->print_cr(" #### error: no base file defined"); tonyp@1479: return; tonyp@1479: } tonyp@1479: tonyp@1479: if (strlen(G1PrintReachableBaseFile) + 1 + strlen(str) > tonyp@1479: (JVM_MAXPATHLEN - 1)) { tonyp@1479: gclog_or_tty->print_cr(" #### error: file name too long"); tonyp@1479: return; tonyp@1479: } tonyp@1479: tonyp@1479: char file_name[JVM_MAXPATHLEN]; tonyp@1479: sprintf(file_name, "%s.%s", G1PrintReachableBaseFile, str); tonyp@1479: gclog_or_tty->print_cr(" dumping to file %s", file_name); tonyp@1479: tonyp@1479: fileStream fout(file_name); tonyp@1479: if (!fout.is_open()) { tonyp@1479: gclog_or_tty->print_cr(" #### error: could not open file"); tonyp@1479: return; tonyp@1479: } tonyp@1479: tonyp@1479: outputStream* out = &fout; tonyp@3957: out->print_cr("-- USING %s", _g1h->top_at_mark_start_str(vo)); tonyp@1479: out->cr(); tonyp@1479: tonyp@1823: out->print_cr("--- ITERATING OVER REGIONS"); tonyp@1479: out->cr(); johnc@2969: PrintReachableRegionClosure rcl(out, vo, all); ysr@777: _g1h->heap_region_iterate(&rcl); tonyp@1479: out->cr(); tonyp@1479: tonyp@1479: gclog_or_tty->print_cr(" done"); tonyp@1823: gclog_or_tty->flush(); ysr@777: } ysr@777: tonyp@1479: #endif // PRODUCT tonyp@1479: tonyp@3416: void ConcurrentMark::clearRangePrevBitmap(MemRegion mr) { ysr@777: // Note we are overriding the read-only view of the prev map here, via ysr@777: // the cast. ysr@777: ((CMBitMap*)_prevMarkBitMap)->clearRange(mr); tonyp@3416: } tonyp@3416: tonyp@3416: void ConcurrentMark::clearRangeNextBitmap(MemRegion mr) { ysr@777: _nextMarkBitMap->clearRange(mr); ysr@777: } ysr@777: tonyp@3416: void ConcurrentMark::clearRangeBothBitmaps(MemRegion mr) { tonyp@3416: clearRangePrevBitmap(mr); tonyp@3416: clearRangeNextBitmap(mr); tonyp@3416: } tonyp@3416: ysr@777: HeapRegion* johnc@4173: ConcurrentMark::claim_region(uint worker_id) { ysr@777: // "checkpoint" the finger ysr@777: HeapWord* finger = _finger; ysr@777: ysr@777: // _heap_end will not change underneath our feet; it only changes at ysr@777: // yield points. ysr@777: while (finger < _heap_end) { tonyp@1458: assert(_g1h->is_in_g1_reserved(finger), "invariant"); ysr@777: tonyp@2968: // Note on how this code handles humongous regions. In the tonyp@2968: // normal case the finger will reach the start of a "starts tonyp@2968: // humongous" (SH) region. Its end will either be the end of the tonyp@2968: // last "continues humongous" (CH) region in the sequence, or the tonyp@2968: // standard end of the SH region (if the SH is the only region in tonyp@2968: // the sequence). That way claim_region() will skip over the CH tonyp@2968: // regions. However, there is a subtle race between a CM thread tonyp@2968: // executing this method and a mutator thread doing a humongous tonyp@2968: // object allocation. The two are not mutually exclusive as the CM tonyp@2968: // thread does not need to hold the Heap_lock when it gets tonyp@2968: // here. So there is a chance that claim_region() will come across tonyp@2968: // a free region that's in the progress of becoming a SH or a CH tonyp@2968: // region. In the former case, it will either tonyp@2968: // a) Miss the update to the region's end, in which case it will tonyp@2968: // visit every subsequent CH region, will find their bitmaps tonyp@2968: // empty, and do nothing, or tonyp@2968: // b) Will observe the update of the region's end (in which case tonyp@2968: // it will skip the subsequent CH regions). tonyp@2968: // If it comes across a region that suddenly becomes CH, the tonyp@2968: // scenario will be similar to b). So, the race between tonyp@2968: // claim_region() and a humongous object allocation might force us tonyp@2968: // to do a bit of unnecessary work (due to some unnecessary bitmap tonyp@2968: // iterations) but it should not introduce and correctness issues. tonyp@2968: HeapRegion* curr_region = _g1h->heap_region_containing_raw(finger); ysr@777: HeapWord* bottom = curr_region->bottom(); ysr@777: HeapWord* end = curr_region->end(); ysr@777: HeapWord* limit = curr_region->next_top_at_mark_start(); ysr@777: tonyp@2968: if (verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] curr_region = "PTR_FORMAT" " ysr@777: "["PTR_FORMAT", "PTR_FORMAT"), " ysr@777: "limit = "PTR_FORMAT, johnc@4173: worker_id, curr_region, bottom, end, limit); tonyp@2968: } tonyp@2968: tonyp@2968: // Is the gap between reading the finger and doing the CAS too long? tonyp@2968: HeapWord* res = (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger); ysr@777: if (res == finger) { ysr@777: // we succeeded ysr@777: ysr@777: // notice that _finger == end cannot be guaranteed here since, ysr@777: // someone else might have moved the finger even further tonyp@1458: assert(_finger >= end, "the finger should have moved forward"); ysr@777: tonyp@2973: if (verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] we were successful with region = " johnc@4173: PTR_FORMAT, worker_id, curr_region); tonyp@2973: } ysr@777: ysr@777: if (limit > bottom) { tonyp@2973: if (verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] region "PTR_FORMAT" is not empty, " johnc@4173: "returning it ", worker_id, curr_region); tonyp@2973: } ysr@777: return curr_region; ysr@777: } else { tonyp@1458: assert(limit == bottom, tonyp@1458: "the region limit should be at bottom"); tonyp@2973: if (verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] region "PTR_FORMAT" is empty, " johnc@4173: "returning NULL", worker_id, curr_region); tonyp@2973: } ysr@777: // we return NULL and the caller should try calling ysr@777: // claim_region() again. ysr@777: return NULL; ysr@777: } ysr@777: } else { tonyp@1458: assert(_finger > finger, "the finger should have moved forward"); tonyp@2973: if (verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] somebody else moved the finger, " ysr@777: "global finger = "PTR_FORMAT", " ysr@777: "our finger = "PTR_FORMAT, johnc@4173: worker_id, _finger, finger); tonyp@2973: } ysr@777: ysr@777: // read it again ysr@777: finger = _finger; ysr@777: } ysr@777: } ysr@777: ysr@777: return NULL; ysr@777: } ysr@777: tonyp@3416: #ifndef PRODUCT tonyp@3416: enum VerifyNoCSetOopsPhase { tonyp@3416: VerifyNoCSetOopsStack, tonyp@3416: VerifyNoCSetOopsQueues, tonyp@3416: VerifyNoCSetOopsSATBCompleted, tonyp@3416: VerifyNoCSetOopsSATBThread tonyp@3416: }; tonyp@3416: tonyp@3416: class VerifyNoCSetOopsClosure : public OopClosure, public ObjectClosure { tonyp@3416: private: tonyp@3416: G1CollectedHeap* _g1h; tonyp@3416: VerifyNoCSetOopsPhase _phase; tonyp@3416: int _info; tonyp@3416: tonyp@3416: const char* phase_str() { tonyp@3416: switch (_phase) { tonyp@3416: case VerifyNoCSetOopsStack: return "Stack"; tonyp@3416: case VerifyNoCSetOopsQueues: return "Queue"; tonyp@3416: case VerifyNoCSetOopsSATBCompleted: return "Completed SATB Buffers"; tonyp@3416: case VerifyNoCSetOopsSATBThread: return "Thread SATB Buffers"; tonyp@3416: default: ShouldNotReachHere(); tonyp@3416: } tonyp@3416: return NULL; ysr@777: } johnc@2190: tonyp@3416: void do_object_work(oop obj) { tonyp@3416: guarantee(!_g1h->obj_in_cs(obj), tonyp@3416: err_msg("obj: "PTR_FORMAT" in CSet, phase: %s, info: %d", tonyp@3416: (void*) obj, phase_str(), _info)); johnc@2190: } johnc@2190: tonyp@3416: public: tonyp@3416: VerifyNoCSetOopsClosure() : _g1h(G1CollectedHeap::heap()) { } tonyp@3416: tonyp@3416: void set_phase(VerifyNoCSetOopsPhase phase, int info = -1) { tonyp@3416: _phase = phase; tonyp@3416: _info = info; tonyp@3416: } tonyp@3416: tonyp@3416: virtual void do_oop(oop* p) { tonyp@3416: oop obj = oopDesc::load_decode_heap_oop(p); tonyp@3416: do_object_work(obj); tonyp@3416: } tonyp@3416: tonyp@3416: virtual void do_oop(narrowOop* p) { tonyp@3416: // We should not come across narrow oops while scanning marking tonyp@3416: // stacks and SATB buffers. tonyp@3416: ShouldNotReachHere(); tonyp@3416: } tonyp@3416: tonyp@3416: virtual void do_object(oop obj) { tonyp@3416: do_object_work(obj); tonyp@3416: } tonyp@3416: }; tonyp@3416: tonyp@3416: void ConcurrentMark::verify_no_cset_oops(bool verify_stacks, tonyp@3416: bool verify_enqueued_buffers, tonyp@3416: bool verify_thread_buffers, tonyp@3416: bool verify_fingers) { tonyp@3416: assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint"); tonyp@3416: if (!G1CollectedHeap::heap()->mark_in_progress()) { tonyp@3416: return; tonyp@3416: } tonyp@3416: tonyp@3416: VerifyNoCSetOopsClosure cl; tonyp@3416: tonyp@3416: if (verify_stacks) { tonyp@3416: // Verify entries on the global mark stack tonyp@3416: cl.set_phase(VerifyNoCSetOopsStack); tonyp@3416: _markStack.oops_do(&cl); tonyp@3416: tonyp@3416: // Verify entries on the task queues johnc@4173: for (uint i = 0; i < _max_worker_id; i += 1) { tonyp@3416: cl.set_phase(VerifyNoCSetOopsQueues, i); johnc@4333: CMTaskQueue* queue = _task_queues->queue(i); tonyp@3416: queue->oops_do(&cl); tonyp@3416: } tonyp@3416: } tonyp@3416: tonyp@3416: SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); tonyp@3416: tonyp@3416: // Verify entries on the enqueued SATB buffers tonyp@3416: if (verify_enqueued_buffers) { tonyp@3416: cl.set_phase(VerifyNoCSetOopsSATBCompleted); tonyp@3416: satb_qs.iterate_completed_buffers_read_only(&cl); tonyp@3416: } tonyp@3416: tonyp@3416: // Verify entries on the per-thread SATB buffers tonyp@3416: if (verify_thread_buffers) { tonyp@3416: cl.set_phase(VerifyNoCSetOopsSATBThread); tonyp@3416: satb_qs.iterate_thread_buffers_read_only(&cl); tonyp@3416: } tonyp@3416: tonyp@3416: if (verify_fingers) { tonyp@3416: // Verify the global finger tonyp@3416: HeapWord* global_finger = finger(); tonyp@3416: if (global_finger != NULL && global_finger < _heap_end) { tonyp@3416: // The global finger always points to a heap region boundary. We tonyp@3416: // use heap_region_containing_raw() to get the containing region tonyp@3416: // given that the global finger could be pointing to a free region tonyp@3416: // which subsequently becomes continues humongous. If that tonyp@3416: // happens, heap_region_containing() will return the bottom of the tonyp@3416: // corresponding starts humongous region and the check below will tonyp@3416: // not hold any more. tonyp@3416: HeapRegion* global_hr = _g1h->heap_region_containing_raw(global_finger); tonyp@3416: guarantee(global_finger == global_hr->bottom(), tonyp@3416: err_msg("global finger: "PTR_FORMAT" region: "HR_FORMAT, tonyp@3416: global_finger, HR_FORMAT_PARAMS(global_hr))); tonyp@3416: } tonyp@3416: tonyp@3416: // Verify the task fingers johnc@4173: assert(parallel_marking_threads() <= _max_worker_id, "sanity"); tonyp@3416: for (int i = 0; i < (int) parallel_marking_threads(); i += 1) { tonyp@3416: CMTask* task = _tasks[i]; tonyp@3416: HeapWord* task_finger = task->finger(); tonyp@3416: if (task_finger != NULL && task_finger < _heap_end) { tonyp@3416: // See above note on the global finger verification. tonyp@3416: HeapRegion* task_hr = _g1h->heap_region_containing_raw(task_finger); tonyp@3416: guarantee(task_finger == task_hr->bottom() || tonyp@3416: !task_hr->in_collection_set(), tonyp@3416: err_msg("task finger: "PTR_FORMAT" region: "HR_FORMAT, tonyp@3416: task_finger, HR_FORMAT_PARAMS(task_hr))); tonyp@3416: } tonyp@3416: } tonyp@3416: } ysr@777: } tonyp@3416: #endif // PRODUCT ysr@777: johnc@3463: // Aggregate the counting data that was constructed concurrently johnc@3463: // with marking. johnc@3463: class AggregateCountDataHRClosure: public HeapRegionClosure { johnc@4123: G1CollectedHeap* _g1h; johnc@3463: ConcurrentMark* _cm; johnc@4123: CardTableModRefBS* _ct_bs; johnc@3463: BitMap* _cm_card_bm; johnc@4173: uint _max_worker_id; johnc@3463: johnc@3463: public: johnc@4123: AggregateCountDataHRClosure(G1CollectedHeap* g1h, johnc@3463: BitMap* cm_card_bm, johnc@4173: uint max_worker_id) : johnc@4123: _g1h(g1h), _cm(g1h->concurrent_mark()), johnc@4123: _ct_bs((CardTableModRefBS*) (g1h->barrier_set())), johnc@4173: _cm_card_bm(cm_card_bm), _max_worker_id(max_worker_id) { } johnc@3463: johnc@3463: bool doHeapRegion(HeapRegion* hr) { johnc@3463: if (hr->continuesHumongous()) { johnc@3463: // We will ignore these here and process them when their johnc@3463: // associated "starts humongous" region is processed. johnc@3463: // Note that we cannot rely on their associated johnc@3463: // "starts humongous" region to have their bit set to 1 johnc@3463: // since, due to the region chunking in the parallel region johnc@3463: // iteration, a "continues humongous" region might be visited johnc@3463: // before its associated "starts humongous". johnc@3463: return false; johnc@3463: } johnc@3463: johnc@3463: HeapWord* start = hr->bottom(); johnc@3463: HeapWord* limit = hr->next_top_at_mark_start(); johnc@3463: HeapWord* end = hr->end(); johnc@3463: johnc@3463: assert(start <= limit && limit <= hr->top() && hr->top() <= hr->end(), johnc@3463: err_msg("Preconditions not met - " johnc@3463: "start: "PTR_FORMAT", limit: "PTR_FORMAT", " johnc@3463: "top: "PTR_FORMAT", end: "PTR_FORMAT, johnc@3463: start, limit, hr->top(), hr->end())); johnc@3463: johnc@3463: assert(hr->next_marked_bytes() == 0, "Precondition"); johnc@3463: johnc@3463: if (start == limit) { johnc@3463: // NTAMS of this region has not been set so nothing to do. johnc@3463: return false; johnc@3463: } johnc@3463: johnc@4123: // 'start' should be in the heap. johnc@4123: assert(_g1h->is_in_g1_reserved(start) && _ct_bs->is_card_aligned(start), "sanity"); johnc@4123: // 'end' *may* be just beyone the end of the heap (if hr is the last region) johnc@4123: assert(!_g1h->is_in_g1_reserved(end) || _ct_bs->is_card_aligned(end), "sanity"); johnc@3463: johnc@3463: BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start); johnc@3463: BitMap::idx_t limit_idx = _cm->card_bitmap_index_for(limit); johnc@3463: BitMap::idx_t end_idx = _cm->card_bitmap_index_for(end); johnc@3463: johnc@4123: // If ntams is not card aligned then we bump card bitmap index johnc@4123: // for limit so that we get the all the cards spanned by johnc@4123: // the object ending at ntams. johnc@4123: // Note: if this is the last region in the heap then ntams johnc@4123: // could be actually just beyond the end of the the heap; johnc@4123: // limit_idx will then correspond to a (non-existent) card johnc@4123: // that is also outside the heap. johnc@4123: if (_g1h->is_in_g1_reserved(limit) && !_ct_bs->is_card_aligned(limit)) { johnc@3463: limit_idx += 1; johnc@3463: } johnc@3463: johnc@3463: assert(limit_idx <= end_idx, "or else use atomics"); johnc@3463: johnc@3463: // Aggregate the "stripe" in the count data associated with hr. tonyp@3713: uint hrs_index = hr->hrs_index(); johnc@3463: size_t marked_bytes = 0; johnc@3463: johnc@4173: for (uint i = 0; i < _max_worker_id; i += 1) { johnc@3463: size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i); johnc@3463: BitMap* task_card_bm = _cm->count_card_bitmap_for(i); johnc@3463: johnc@3463: // Fetch the marked_bytes in this region for task i and johnc@3463: // add it to the running total for this region. johnc@3463: marked_bytes += marked_bytes_array[hrs_index]; johnc@3463: johnc@4173: // Now union the bitmaps[0,max_worker_id)[start_idx..limit_idx) johnc@3463: // into the global card bitmap. johnc@3463: BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx); johnc@3463: johnc@3463: while (scan_idx < limit_idx) { johnc@3463: assert(task_card_bm->at(scan_idx) == true, "should be"); johnc@3463: _cm_card_bm->set_bit(scan_idx); johnc@3463: assert(_cm_card_bm->at(scan_idx) == true, "should be"); johnc@3463: johnc@3463: // BitMap::get_next_one_offset() can handle the case when johnc@3463: // its left_offset parameter is greater than its right_offset johnc@4123: // parameter. It does, however, have an early exit if johnc@3463: // left_offset == right_offset. So let's limit the value johnc@3463: // passed in for left offset here. johnc@3463: BitMap::idx_t next_idx = MIN2(scan_idx + 1, limit_idx); johnc@3463: scan_idx = task_card_bm->get_next_one_offset(next_idx, limit_idx); johnc@3463: } johnc@3463: } johnc@3463: johnc@3463: // Update the marked bytes for this region. johnc@3463: hr->add_to_marked_bytes(marked_bytes); johnc@3463: johnc@3463: // Next heap region johnc@3463: return false; johnc@3463: } johnc@3463: }; johnc@3463: johnc@3463: class G1AggregateCountDataTask: public AbstractGangTask { johnc@3463: protected: johnc@3463: G1CollectedHeap* _g1h; johnc@3463: ConcurrentMark* _cm; johnc@3463: BitMap* _cm_card_bm; johnc@4173: uint _max_worker_id; johnc@3463: int _active_workers; johnc@3463: johnc@3463: public: johnc@3463: G1AggregateCountDataTask(G1CollectedHeap* g1h, johnc@3463: ConcurrentMark* cm, johnc@3463: BitMap* cm_card_bm, johnc@4173: uint max_worker_id, johnc@3463: int n_workers) : johnc@3463: AbstractGangTask("Count Aggregation"), johnc@3463: _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm), johnc@4173: _max_worker_id(max_worker_id), johnc@3463: _active_workers(n_workers) { } johnc@3463: johnc@3463: void work(uint worker_id) { johnc@4173: AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_worker_id); johnc@3463: johnc@3463: if (G1CollectedHeap::use_parallel_gc_threads()) { johnc@3463: _g1h->heap_region_par_iterate_chunked(&cl, worker_id, johnc@3463: _active_workers, johnc@3463: HeapRegion::AggregateCountClaimValue); johnc@3463: } else { johnc@3463: _g1h->heap_region_iterate(&cl); johnc@3463: } johnc@3463: } johnc@3463: }; johnc@3463: johnc@3463: johnc@3463: void ConcurrentMark::aggregate_count_data() { johnc@3463: int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ? johnc@3463: _g1h->workers()->active_workers() : johnc@3463: 1); johnc@3463: johnc@3463: G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm, johnc@4173: _max_worker_id, n_workers); johnc@3463: johnc@3463: if (G1CollectedHeap::use_parallel_gc_threads()) { johnc@3463: assert(_g1h->check_heap_region_claim_values(HeapRegion::InitialClaimValue), johnc@3463: "sanity check"); johnc@3463: _g1h->set_par_threads(n_workers); johnc@3463: _g1h->workers()->run_task(&g1_par_agg_task); johnc@3463: _g1h->set_par_threads(0); johnc@3463: johnc@3463: assert(_g1h->check_heap_region_claim_values(HeapRegion::AggregateCountClaimValue), johnc@3463: "sanity check"); johnc@3463: _g1h->reset_heap_region_claim_values(); johnc@3463: } else { johnc@3463: g1_par_agg_task.work(0); johnc@3463: } johnc@3463: } johnc@3463: johnc@3463: // Clear the per-worker arrays used to store the per-region counting data johnc@3463: void ConcurrentMark::clear_all_count_data() { johnc@3463: // Clear the global card bitmap - it will be filled during johnc@3463: // liveness count aggregation (during remark) and the johnc@3463: // final counting task. johnc@3463: _card_bm.clear(); johnc@3463: johnc@3463: // Clear the global region bitmap - it will be filled as part johnc@3463: // of the final counting task. johnc@3463: _region_bm.clear(); johnc@3463: tonyp@3713: uint max_regions = _g1h->max_regions(); johnc@4173: assert(_max_worker_id > 0, "uninitialized"); johnc@4173: johnc@4173: for (uint i = 0; i < _max_worker_id; i += 1) { johnc@3463: BitMap* task_card_bm = count_card_bitmap_for(i); johnc@3463: size_t* marked_bytes_array = count_marked_bytes_array_for(i); johnc@3463: johnc@3463: assert(task_card_bm->size() == _card_bm.size(), "size mismatch"); johnc@3463: assert(marked_bytes_array != NULL, "uninitialized"); johnc@3463: tonyp@3713: memset(marked_bytes_array, 0, (size_t) max_regions * sizeof(size_t)); johnc@3463: task_card_bm->clear(); johnc@3463: } johnc@3463: } johnc@3463: ysr@777: void ConcurrentMark::print_stats() { ysr@777: if (verbose_stats()) { ysr@777: gclog_or_tty->print_cr("---------------------------------------------------------------------"); ysr@777: for (size_t i = 0; i < _active_tasks; ++i) { ysr@777: _tasks[i]->print_stats(); ysr@777: gclog_or_tty->print_cr("---------------------------------------------------------------------"); ysr@777: } ysr@777: } ysr@777: } ysr@777: ysr@777: // abandon current marking iteration due to a Full GC ysr@777: void ConcurrentMark::abort() { ysr@777: // Clear all marks to force marking thread to do nothing ysr@777: _nextMarkBitMap->clearAll(); johnc@3463: // Clear the liveness counting data johnc@3463: clear_all_count_data(); ysr@777: // Empty mark stack johnc@4386: reset_marking_state(); johnc@4173: for (uint i = 0; i < _max_worker_id; ++i) { ysr@777: _tasks[i]->clear_region_fields(); johnc@2190: } ysr@777: _has_aborted = true; ysr@777: ysr@777: SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); ysr@777: satb_mq_set.abandon_partial_marking(); tonyp@1752: // This can be called either during or outside marking, we'll read tonyp@1752: // the expected_active value from the SATB queue set. tonyp@1752: satb_mq_set.set_active_all_threads( tonyp@1752: false, /* new active value */ tonyp@1752: satb_mq_set.is_active() /* expected_active */); sla@5237: sla@5237: _g1h->trace_heap_after_concurrent_cycle(); sla@5237: _g1h->register_concurrent_cycle_end(); ysr@777: } ysr@777: ysr@777: static void print_ms_time_info(const char* prefix, const char* name, ysr@777: NumberSeq& ns) { ysr@777: gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).", ysr@777: prefix, ns.num(), name, ns.sum()/1000.0, ns.avg()); ysr@777: if (ns.num() > 0) { ysr@777: gclog_or_tty->print_cr("%s [std. dev = %8.2f ms, max = %8.2f ms]", ysr@777: prefix, ns.sd(), ns.maximum()); ysr@777: } ysr@777: } ysr@777: ysr@777: void ConcurrentMark::print_summary_info() { ysr@777: gclog_or_tty->print_cr(" Concurrent marking:"); ysr@777: print_ms_time_info(" ", "init marks", _init_times); ysr@777: print_ms_time_info(" ", "remarks", _remark_times); ysr@777: { ysr@777: print_ms_time_info(" ", "final marks", _remark_mark_times); ysr@777: print_ms_time_info(" ", "weak refs", _remark_weak_ref_times); ysr@777: ysr@777: } ysr@777: print_ms_time_info(" ", "cleanups", _cleanup_times); ysr@777: gclog_or_tty->print_cr(" Final counting total time = %8.2f s (avg = %8.2f ms).", ysr@777: _total_counting_time, ysr@777: (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / ysr@777: (double)_cleanup_times.num() ysr@777: : 0.0)); ysr@777: if (G1ScrubRemSets) { ysr@777: gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).", ysr@777: _total_rs_scrub_time, ysr@777: (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / ysr@777: (double)_cleanup_times.num() ysr@777: : 0.0)); ysr@777: } ysr@777: gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.", ysr@777: (_init_times.sum() + _remark_times.sum() + ysr@777: _cleanup_times.sum())/1000.0); ysr@777: gclog_or_tty->print_cr(" Total concurrent time = %8.2f s " johnc@3463: "(%8.2f s marking).", ysr@777: cmThread()->vtime_accum(), johnc@3463: cmThread()->vtime_mark_accum()); ysr@777: } ysr@777: tonyp@1454: void ConcurrentMark::print_worker_threads_on(outputStream* st) const { johnc@4549: if (use_parallel_marking_threads()) { johnc@4549: _parallel_workers->print_worker_threads_on(st); johnc@4549: } tonyp@1454: } tonyp@1454: stefank@4904: void ConcurrentMark::print_on_error(outputStream* st) const { stefank@4904: st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT, stefank@4904: _prevMarkBitMap, _nextMarkBitMap); stefank@4904: _prevMarkBitMap->print_on_error(st, " Prev Bits: "); stefank@4904: _nextMarkBitMap->print_on_error(st, " Next Bits: "); stefank@4904: } stefank@4904: ysr@777: // We take a break if someone is trying to stop the world. jmasa@3357: bool ConcurrentMark::do_yield_check(uint worker_id) { ysr@777: if (should_yield()) { jmasa@3357: if (worker_id == 0) { ysr@777: _g1h->g1_policy()->record_concurrent_pause(); tonyp@2973: } ysr@777: cmThread()->yield(); ysr@777: return true; ysr@777: } else { ysr@777: return false; ysr@777: } ysr@777: } ysr@777: ysr@777: bool ConcurrentMark::should_yield() { ysr@777: return cmThread()->should_yield(); ysr@777: } ysr@777: ysr@777: bool ConcurrentMark::containing_card_is_marked(void* p) { ysr@777: size_t offset = pointer_delta(p, _g1h->reserved_region().start(), 1); ysr@777: return _card_bm.at(offset >> CardTableModRefBS::card_shift); ysr@777: } ysr@777: ysr@777: bool ConcurrentMark::containing_cards_are_marked(void* start, ysr@777: void* last) { tonyp@2973: return containing_card_is_marked(start) && tonyp@2973: containing_card_is_marked(last); ysr@777: } ysr@777: ysr@777: #ifndef PRODUCT ysr@777: // for debugging purposes ysr@777: void ConcurrentMark::print_finger() { ysr@777: gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT, ysr@777: _heap_start, _heap_end, _finger); johnc@4173: for (uint i = 0; i < _max_worker_id; ++i) { johnc@4173: gclog_or_tty->print(" %u: "PTR_FORMAT, i, _tasks[i]->finger()); ysr@777: } ysr@777: gclog_or_tty->print_cr(""); ysr@777: } ysr@777: #endif ysr@777: tonyp@2968: void CMTask::scan_object(oop obj) { tonyp@2968: assert(_nextMarkBitMap->isMarked((HeapWord*) obj), "invariant"); tonyp@2968: tonyp@2968: if (_cm->verbose_high()) { johnc@4173: gclog_or_tty->print_cr("[%u] we're scanning object "PTR_FORMAT, johnc@4173: _worker_id, (void*) obj); tonyp@2968: } tonyp@2968: tonyp@2968: size_t obj_size = obj->size(); tonyp@2968: _words_scanned += obj_size; tonyp@2968: tonyp@2968: obj->oop_iterate(_cm_oop_closure); tonyp@2968: statsOnly( ++_objs_scanned ); tonyp@2968: check_limits(); tonyp@2968: } tonyp@2968: ysr@777: // Closure for iteration over bitmaps ysr@777: class CMBitMapClosure : public BitMapClosure { ysr@777: private: ysr@777: // the bitmap that is being iterated over ysr@777: CMBitMap* _nextMarkBitMap; ysr@777: ConcurrentMark* _cm; ysr@777: CMTask* _task; ysr@777: ysr@777: public: tonyp@3691: CMBitMapClosure(CMTask *task, ConcurrentMark* cm, CMBitMap* nextMarkBitMap) : tonyp@3691: _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { } ysr@777: ysr@777: bool do_bit(size_t offset) { ysr@777: HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset); tonyp@1458: assert(_nextMarkBitMap->isMarked(addr), "invariant"); tonyp@1458: assert( addr < _cm->finger(), "invariant"); ysr@777: tonyp@3691: statsOnly( _task->increase_objs_found_on_bitmap() ); tonyp@3691: assert(addr >= _task->finger(), "invariant"); tonyp@3691: tonyp@3691: // We move that task's local finger along. tonyp@3691: _task->move_finger_to(addr); ysr@777: ysr@777: _task->scan_object(oop(addr)); ysr@777: // we only partially drain the local queue and global stack ysr@777: _task->drain_local_queue(true); ysr@777: _task->drain_global_stack(true); ysr@777: ysr@777: // if the has_aborted flag has been raised, we need to bail out of ysr@777: // the iteration ysr@777: return !_task->has_aborted(); ysr@777: } ysr@777: }; ysr@777: ysr@777: // Closure for iterating over objects, currently only used for ysr@777: // processing SATB buffers. ysr@777: class CMObjectClosure : public ObjectClosure { ysr@777: private: ysr@777: CMTask* _task; ysr@777: ysr@777: public: ysr@777: void do_object(oop obj) { ysr@777: _task->deal_with_reference(obj); ysr@777: } ysr@777: ysr@777: CMObjectClosure(CMTask* task) : _task(task) { } ysr@777: }; ysr@777: tonyp@2968: G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h, tonyp@2968: ConcurrentMark* cm, tonyp@2968: CMTask* task) tonyp@2968: : _g1h(g1h), _cm(cm), _task(task) { tonyp@2968: assert(_ref_processor == NULL, "should be initialized to NULL"); tonyp@2968: tonyp@2968: if (G1UseConcMarkReferenceProcessing) { johnc@3175: _ref_processor = g1h->ref_processor_cm(); tonyp@2968: assert(_ref_processor != NULL, "should not be NULL"); ysr@777: } tonyp@2968: } ysr@777: ysr@777: void CMTask::setup_for_region(HeapRegion* hr) { tonyp@1458: // Separated the asserts so that we know which one fires. tonyp@1458: assert(hr != NULL, tonyp@1458: "claim_region() should have filtered out continues humongous regions"); tonyp@1458: assert(!hr->continuesHumongous(), tonyp@1458: "claim_region() should have filtered out continues humongous regions"); ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] setting up for region "PTR_FORMAT, johnc@4173: _worker_id, hr); tonyp@2973: } ysr@777: ysr@777: _curr_region = hr; ysr@777: _finger = hr->bottom(); ysr@777: update_region_limit(); ysr@777: } ysr@777: ysr@777: void CMTask::update_region_limit() { ysr@777: HeapRegion* hr = _curr_region; ysr@777: HeapWord* bottom = hr->bottom(); ysr@777: HeapWord* limit = hr->next_top_at_mark_start(); ysr@777: ysr@777: if (limit == bottom) { tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] found an empty region " ysr@777: "["PTR_FORMAT", "PTR_FORMAT")", johnc@4173: _worker_id, bottom, limit); tonyp@2973: } ysr@777: // The region was collected underneath our feet. ysr@777: // We set the finger to bottom to ensure that the bitmap ysr@777: // iteration that will follow this will not do anything. ysr@777: // (this is not a condition that holds when we set the region up, ysr@777: // as the region is not supposed to be empty in the first place) ysr@777: _finger = bottom; ysr@777: } else if (limit >= _region_limit) { tonyp@1458: assert(limit >= _finger, "peace of mind"); ysr@777: } else { tonyp@1458: assert(limit < _region_limit, "only way to get here"); ysr@777: // This can happen under some pretty unusual circumstances. An ysr@777: // evacuation pause empties the region underneath our feet (NTAMS ysr@777: // at bottom). We then do some allocation in the region (NTAMS ysr@777: // stays at bottom), followed by the region being used as a GC ysr@777: // alloc region (NTAMS will move to top() and the objects ysr@777: // originally below it will be grayed). All objects now marked in ysr@777: // the region are explicitly grayed, if below the global finger, ysr@777: // and we do not need in fact to scan anything else. So, we simply ysr@777: // set _finger to be limit to ensure that the bitmap iteration ysr@777: // doesn't do anything. ysr@777: _finger = limit; ysr@777: } ysr@777: ysr@777: _region_limit = limit; ysr@777: } ysr@777: ysr@777: void CMTask::giveup_current_region() { tonyp@1458: assert(_curr_region != NULL, "invariant"); tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] giving up region "PTR_FORMAT, johnc@4173: _worker_id, _curr_region); tonyp@2973: } ysr@777: clear_region_fields(); ysr@777: } ysr@777: ysr@777: void CMTask::clear_region_fields() { ysr@777: // Values for these three fields that indicate that we're not ysr@777: // holding on to a region. ysr@777: _curr_region = NULL; ysr@777: _finger = NULL; ysr@777: _region_limit = NULL; ysr@777: } ysr@777: tonyp@2968: void CMTask::set_cm_oop_closure(G1CMOopClosure* cm_oop_closure) { tonyp@2968: if (cm_oop_closure == NULL) { tonyp@2968: assert(_cm_oop_closure != NULL, "invariant"); tonyp@2968: } else { tonyp@2968: assert(_cm_oop_closure == NULL, "invariant"); tonyp@2968: } tonyp@2968: _cm_oop_closure = cm_oop_closure; tonyp@2968: } tonyp@2968: ysr@777: void CMTask::reset(CMBitMap* nextMarkBitMap) { tonyp@1458: guarantee(nextMarkBitMap != NULL, "invariant"); ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] resetting", _worker_id); tonyp@2973: } ysr@777: ysr@777: _nextMarkBitMap = nextMarkBitMap; ysr@777: clear_region_fields(); ysr@777: ysr@777: _calls = 0; ysr@777: _elapsed_time_ms = 0.0; ysr@777: _termination_time_ms = 0.0; ysr@777: _termination_start_time_ms = 0.0; ysr@777: ysr@777: #if _MARKING_STATS_ ysr@777: _local_pushes = 0; ysr@777: _local_pops = 0; ysr@777: _local_max_size = 0; ysr@777: _objs_scanned = 0; ysr@777: _global_pushes = 0; ysr@777: _global_pops = 0; ysr@777: _global_max_size = 0; ysr@777: _global_transfers_to = 0; ysr@777: _global_transfers_from = 0; ysr@777: _regions_claimed = 0; ysr@777: _objs_found_on_bitmap = 0; ysr@777: _satb_buffers_processed = 0; ysr@777: _steal_attempts = 0; ysr@777: _steals = 0; ysr@777: _aborted = 0; ysr@777: _aborted_overflow = 0; ysr@777: _aborted_cm_aborted = 0; ysr@777: _aborted_yield = 0; ysr@777: _aborted_timed_out = 0; ysr@777: _aborted_satb = 0; ysr@777: _aborted_termination = 0; ysr@777: #endif // _MARKING_STATS_ ysr@777: } ysr@777: ysr@777: bool CMTask::should_exit_termination() { ysr@777: regular_clock_call(); ysr@777: // This is called when we are in the termination protocol. We should ysr@777: // quit if, for some reason, this task wants to abort or the global ysr@777: // stack is not empty (this means that we can get work from it). ysr@777: return !_cm->mark_stack_empty() || has_aborted(); ysr@777: } ysr@777: ysr@777: void CMTask::reached_limit() { tonyp@1458: assert(_words_scanned >= _words_scanned_limit || tonyp@1458: _refs_reached >= _refs_reached_limit , tonyp@1458: "shouldn't have been called otherwise"); ysr@777: regular_clock_call(); ysr@777: } ysr@777: ysr@777: void CMTask::regular_clock_call() { tonyp@2973: if (has_aborted()) return; ysr@777: ysr@777: // First, we need to recalculate the words scanned and refs reached ysr@777: // limits for the next clock call. ysr@777: recalculate_limits(); ysr@777: ysr@777: // During the regular clock call we do the following ysr@777: ysr@777: // (1) If an overflow has been flagged, then we abort. ysr@777: if (_cm->has_overflown()) { ysr@777: set_has_aborted(); ysr@777: return; ysr@777: } ysr@777: ysr@777: // If we are not concurrent (i.e. we're doing remark) we don't need ysr@777: // to check anything else. The other steps are only needed during ysr@777: // the concurrent marking phase. tonyp@2973: if (!concurrent()) return; ysr@777: ysr@777: // (2) If marking has been aborted for Full GC, then we also abort. ysr@777: if (_cm->has_aborted()) { ysr@777: set_has_aborted(); ysr@777: statsOnly( ++_aborted_cm_aborted ); ysr@777: return; ysr@777: } ysr@777: ysr@777: double curr_time_ms = os::elapsedVTime() * 1000.0; ysr@777: ysr@777: // (3) If marking stats are enabled, then we update the step history. ysr@777: #if _MARKING_STATS_ tonyp@2973: if (_words_scanned >= _words_scanned_limit) { ysr@777: ++_clock_due_to_scanning; tonyp@2973: } tonyp@2973: if (_refs_reached >= _refs_reached_limit) { ysr@777: ++_clock_due_to_marking; tonyp@2973: } ysr@777: ysr@777: double last_interval_ms = curr_time_ms - _interval_start_time_ms; ysr@777: _interval_start_time_ms = curr_time_ms; ysr@777: _all_clock_intervals_ms.add(last_interval_ms); ysr@777: ysr@777: if (_cm->verbose_medium()) { johnc@4173: gclog_or_tty->print_cr("[%u] regular clock, interval = %1.2lfms, " tonyp@2973: "scanned = %d%s, refs reached = %d%s", johnc@4173: _worker_id, last_interval_ms, tonyp@2973: _words_scanned, tonyp@2973: (_words_scanned >= _words_scanned_limit) ? " (*)" : "", tonyp@2973: _refs_reached, tonyp@2973: (_refs_reached >= _refs_reached_limit) ? " (*)" : ""); ysr@777: } ysr@777: #endif // _MARKING_STATS_ ysr@777: ysr@777: // (4) We check whether we should yield. If we have to, then we abort. ysr@777: if (_cm->should_yield()) { ysr@777: // We should yield. To do this we abort the task. The caller is ysr@777: // responsible for yielding. ysr@777: set_has_aborted(); ysr@777: statsOnly( ++_aborted_yield ); ysr@777: return; ysr@777: } ysr@777: ysr@777: // (5) We check whether we've reached our time quota. If we have, ysr@777: // then we abort. ysr@777: double elapsed_time_ms = curr_time_ms - _start_time_ms; ysr@777: if (elapsed_time_ms > _time_target_ms) { ysr@777: set_has_aborted(); johnc@2494: _has_timed_out = true; ysr@777: statsOnly( ++_aborted_timed_out ); ysr@777: return; ysr@777: } ysr@777: ysr@777: // (6) Finally, we check whether there are enough completed STAB ysr@777: // buffers available for processing. If there are, we abort. ysr@777: SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); ysr@777: if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) { tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] aborting to deal with pending SATB buffers", johnc@4173: _worker_id); tonyp@2973: } ysr@777: // we do need to process SATB buffers, we'll abort and restart ysr@777: // the marking task to do so ysr@777: set_has_aborted(); ysr@777: statsOnly( ++_aborted_satb ); ysr@777: return; ysr@777: } ysr@777: } ysr@777: ysr@777: void CMTask::recalculate_limits() { ysr@777: _real_words_scanned_limit = _words_scanned + words_scanned_period; ysr@777: _words_scanned_limit = _real_words_scanned_limit; ysr@777: ysr@777: _real_refs_reached_limit = _refs_reached + refs_reached_period; ysr@777: _refs_reached_limit = _real_refs_reached_limit; ysr@777: } ysr@777: ysr@777: void CMTask::decrease_limits() { ysr@777: // This is called when we believe that we're going to do an infrequent ysr@777: // operation which will increase the per byte scanned cost (i.e. move ysr@777: // entries to/from the global stack). It basically tries to decrease the ysr@777: // scanning limit so that the clock is called earlier. ysr@777: tonyp@2973: if (_cm->verbose_medium()) { johnc@4173: gclog_or_tty->print_cr("[%u] decreasing limits", _worker_id); tonyp@2973: } ysr@777: ysr@777: _words_scanned_limit = _real_words_scanned_limit - ysr@777: 3 * words_scanned_period / 4; ysr@777: _refs_reached_limit = _real_refs_reached_limit - ysr@777: 3 * refs_reached_period / 4; ysr@777: } ysr@777: ysr@777: void CMTask::move_entries_to_global_stack() { ysr@777: // local array where we'll store the entries that will be popped ysr@777: // from the local queue ysr@777: oop buffer[global_stack_transfer_size]; ysr@777: ysr@777: int n = 0; ysr@777: oop obj; ysr@777: while (n < global_stack_transfer_size && _task_queue->pop_local(obj)) { ysr@777: buffer[n] = obj; ysr@777: ++n; ysr@777: } ysr@777: ysr@777: if (n > 0) { ysr@777: // we popped at least one entry from the local queue ysr@777: ysr@777: statsOnly( ++_global_transfers_to; _local_pops += n ); ysr@777: ysr@777: if (!_cm->mark_stack_push(buffer, n)) { tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] aborting due to global stack overflow", johnc@4173: _worker_id); tonyp@2973: } ysr@777: set_has_aborted(); ysr@777: } else { ysr@777: // the transfer was successful ysr@777: tonyp@2973: if (_cm->verbose_medium()) { johnc@4173: gclog_or_tty->print_cr("[%u] pushed %d entries to the global stack", johnc@4173: _worker_id, n); tonyp@2973: } ysr@777: statsOnly( int tmp_size = _cm->mark_stack_size(); tonyp@2973: if (tmp_size > _global_max_size) { ysr@777: _global_max_size = tmp_size; tonyp@2973: } ysr@777: _global_pushes += n ); ysr@777: } ysr@777: } ysr@777: ysr@777: // this operation was quite expensive, so decrease the limits ysr@777: decrease_limits(); ysr@777: } ysr@777: ysr@777: void CMTask::get_entries_from_global_stack() { ysr@777: // local array where we'll store the entries that will be popped ysr@777: // from the global stack. ysr@777: oop buffer[global_stack_transfer_size]; ysr@777: int n; ysr@777: _cm->mark_stack_pop(buffer, global_stack_transfer_size, &n); tonyp@1458: assert(n <= global_stack_transfer_size, tonyp@1458: "we should not pop more than the given limit"); ysr@777: if (n > 0) { ysr@777: // yes, we did actually pop at least one entry ysr@777: ysr@777: statsOnly( ++_global_transfers_from; _global_pops += n ); tonyp@2973: if (_cm->verbose_medium()) { johnc@4173: gclog_or_tty->print_cr("[%u] popped %d entries from the global stack", johnc@4173: _worker_id, n); tonyp@2973: } ysr@777: for (int i = 0; i < n; ++i) { ysr@777: bool success = _task_queue->push(buffer[i]); ysr@777: // We only call this when the local queue is empty or under a ysr@777: // given target limit. So, we do not expect this push to fail. tonyp@1458: assert(success, "invariant"); ysr@777: } ysr@777: ysr@777: statsOnly( int tmp_size = _task_queue->size(); tonyp@2973: if (tmp_size > _local_max_size) { ysr@777: _local_max_size = tmp_size; tonyp@2973: } ysr@777: _local_pushes += n ); ysr@777: } ysr@777: ysr@777: // this operation was quite expensive, so decrease the limits ysr@777: decrease_limits(); ysr@777: } ysr@777: ysr@777: void CMTask::drain_local_queue(bool partially) { tonyp@2973: if (has_aborted()) return; ysr@777: ysr@777: // Decide what the target size is, depending whether we're going to ysr@777: // drain it partially (so that other tasks can steal if they run out ysr@777: // of things to do) or totally (at the very end). ysr@777: size_t target_size; tonyp@2973: if (partially) { ysr@777: target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize); tonyp@2973: } else { ysr@777: target_size = 0; tonyp@2973: } ysr@777: ysr@777: if (_task_queue->size() > target_size) { tonyp@2973: if (_cm->verbose_high()) { johnc@4173: gclog_or_tty->print_cr("[%u] draining local queue, target size = %d", johnc@4173: _worker_id, target_size); tonyp@2973: } ysr@777: ysr@777: oop obj; ysr@777: bool ret = _task_queue->pop_local(obj); ysr@777: while (ret) { ysr@777: statsOnly( ++_local_pops ); ysr@777: tonyp@2973: if (_cm->verbose_high()) { johnc@4173: gclog_or_tty->print_cr("[%u] popped "PTR_FORMAT, _worker_id, ysr@777: (void*) obj); tonyp@2973: } ysr@777: tonyp@1458: assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" ); tonyp@2643: assert(!_g1h->is_on_master_free_list( tonyp@2472: _g1h->heap_region_containing((HeapWord*) obj)), "invariant"); ysr@777: ysr@777: scan_object(obj); ysr@777: tonyp@2973: if (_task_queue->size() <= target_size || has_aborted()) { ysr@777: ret = false; tonyp@2973: } else { ysr@777: ret = _task_queue->pop_local(obj); tonyp@2973: } ysr@777: } ysr@777: tonyp@2973: if (_cm->verbose_high()) { johnc@4173: gclog_or_tty->print_cr("[%u] drained local queue, size = %d", johnc@4173: _worker_id, _task_queue->size()); tonyp@2973: } ysr@777: } ysr@777: } ysr@777: ysr@777: void CMTask::drain_global_stack(bool partially) { tonyp@2973: if (has_aborted()) return; ysr@777: ysr@777: // We have a policy to drain the local queue before we attempt to ysr@777: // drain the global stack. tonyp@1458: assert(partially || _task_queue->size() == 0, "invariant"); ysr@777: ysr@777: // Decide what the target size is, depending whether we're going to ysr@777: // drain it partially (so that other tasks can steal if they run out ysr@777: // of things to do) or totally (at the very end). Notice that, ysr@777: // because we move entries from the global stack in chunks or ysr@777: // because another task might be doing the same, we might in fact ysr@777: // drop below the target. But, this is not a problem. ysr@777: size_t target_size; tonyp@2973: if (partially) { ysr@777: target_size = _cm->partial_mark_stack_size_target(); tonyp@2973: } else { ysr@777: target_size = 0; tonyp@2973: } ysr@777: ysr@777: if (_cm->mark_stack_size() > target_size) { tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] draining global_stack, target size %d", johnc@4173: _worker_id, target_size); tonyp@2973: } ysr@777: ysr@777: while (!has_aborted() && _cm->mark_stack_size() > target_size) { ysr@777: get_entries_from_global_stack(); ysr@777: drain_local_queue(partially); ysr@777: } ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] drained global stack, size = %d", johnc@4173: _worker_id, _cm->mark_stack_size()); tonyp@2973: } ysr@777: } ysr@777: } ysr@777: ysr@777: // SATB Queue has several assumptions on whether to call the par or ysr@777: // non-par versions of the methods. this is why some of the code is ysr@777: // replicated. We should really get rid of the single-threaded version ysr@777: // of the code to simplify things. ysr@777: void CMTask::drain_satb_buffers() { tonyp@2973: if (has_aborted()) return; ysr@777: ysr@777: // We set this so that the regular clock knows that we're in the ysr@777: // middle of draining buffers and doesn't set the abort flag when it ysr@777: // notices that SATB buffers are available for draining. It'd be ysr@777: // very counter productive if it did that. :-) ysr@777: _draining_satb_buffers = true; ysr@777: ysr@777: CMObjectClosure oc(this); ysr@777: SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); tonyp@2973: if (G1CollectedHeap::use_parallel_gc_threads()) { johnc@4173: satb_mq_set.set_par_closure(_worker_id, &oc); tonyp@2973: } else { ysr@777: satb_mq_set.set_closure(&oc); tonyp@2973: } ysr@777: ysr@777: // This keeps claiming and applying the closure to completed buffers ysr@777: // until we run out of buffers or we need to abort. jmasa@2188: if (G1CollectedHeap::use_parallel_gc_threads()) { ysr@777: while (!has_aborted() && johnc@4173: satb_mq_set.par_apply_closure_to_completed_buffer(_worker_id)) { tonyp@2973: if (_cm->verbose_medium()) { johnc@4173: gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id); tonyp@2973: } ysr@777: statsOnly( ++_satb_buffers_processed ); ysr@777: regular_clock_call(); ysr@777: } ysr@777: } else { ysr@777: while (!has_aborted() && ysr@777: satb_mq_set.apply_closure_to_completed_buffer()) { tonyp@2973: if (_cm->verbose_medium()) { johnc@4173: gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id); tonyp@2973: } ysr@777: statsOnly( ++_satb_buffers_processed ); ysr@777: regular_clock_call(); ysr@777: } ysr@777: } ysr@777: ysr@777: if (!concurrent() && !has_aborted()) { ysr@777: // We should only do this during remark. tonyp@2973: if (G1CollectedHeap::use_parallel_gc_threads()) { johnc@4173: satb_mq_set.par_iterate_closure_all_threads(_worker_id); tonyp@2973: } else { ysr@777: satb_mq_set.iterate_closure_all_threads(); tonyp@2973: } ysr@777: } ysr@777: ysr@777: _draining_satb_buffers = false; ysr@777: tonyp@1458: assert(has_aborted() || tonyp@1458: concurrent() || tonyp@1458: satb_mq_set.completed_buffers_num() == 0, "invariant"); ysr@777: tonyp@2973: if (G1CollectedHeap::use_parallel_gc_threads()) { johnc@4173: satb_mq_set.set_par_closure(_worker_id, NULL); tonyp@2973: } else { ysr@777: satb_mq_set.set_closure(NULL); tonyp@2973: } ysr@777: ysr@777: // again, this was a potentially expensive operation, decrease the ysr@777: // limits to get the regular clock call early ysr@777: decrease_limits(); ysr@777: } ysr@777: ysr@777: void CMTask::print_stats() { johnc@4173: gclog_or_tty->print_cr("Marking Stats, task = %u, calls = %d", johnc@4173: _worker_id, _calls); ysr@777: gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms", ysr@777: _elapsed_time_ms, _termination_time_ms); ysr@777: gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", ysr@777: _step_times_ms.num(), _step_times_ms.avg(), ysr@777: _step_times_ms.sd()); ysr@777: gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms", ysr@777: _step_times_ms.maximum(), _step_times_ms.sum()); ysr@777: ysr@777: #if _MARKING_STATS_ ysr@777: gclog_or_tty->print_cr(" Clock Intervals (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", ysr@777: _all_clock_intervals_ms.num(), _all_clock_intervals_ms.avg(), ysr@777: _all_clock_intervals_ms.sd()); ysr@777: gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms", ysr@777: _all_clock_intervals_ms.maximum(), ysr@777: _all_clock_intervals_ms.sum()); ysr@777: gclog_or_tty->print_cr(" Clock Causes (cum): scanning = %d, marking = %d", ysr@777: _clock_due_to_scanning, _clock_due_to_marking); ysr@777: gclog_or_tty->print_cr(" Objects: scanned = %d, found on the bitmap = %d", ysr@777: _objs_scanned, _objs_found_on_bitmap); ysr@777: gclog_or_tty->print_cr(" Local Queue: pushes = %d, pops = %d, max size = %d", ysr@777: _local_pushes, _local_pops, _local_max_size); ysr@777: gclog_or_tty->print_cr(" Global Stack: pushes = %d, pops = %d, max size = %d", ysr@777: _global_pushes, _global_pops, _global_max_size); ysr@777: gclog_or_tty->print_cr(" transfers to = %d, transfers from = %d", ysr@777: _global_transfers_to,_global_transfers_from); tonyp@3691: gclog_or_tty->print_cr(" Regions: claimed = %d", _regions_claimed); ysr@777: gclog_or_tty->print_cr(" SATB buffers: processed = %d", _satb_buffers_processed); ysr@777: gclog_or_tty->print_cr(" Steals: attempts = %d, successes = %d", ysr@777: _steal_attempts, _steals); ysr@777: gclog_or_tty->print_cr(" Aborted: %d, due to", _aborted); ysr@777: gclog_or_tty->print_cr(" overflow: %d, global abort: %d, yield: %d", ysr@777: _aborted_overflow, _aborted_cm_aborted, _aborted_yield); ysr@777: gclog_or_tty->print_cr(" time out: %d, SATB: %d, termination: %d", ysr@777: _aborted_timed_out, _aborted_satb, _aborted_termination); ysr@777: #endif // _MARKING_STATS_ ysr@777: } ysr@777: ysr@777: /***************************************************************************** ysr@777: johnc@4787: The do_marking_step(time_target_ms, ...) method is the building johnc@4787: block of the parallel marking framework. It can be called in parallel ysr@777: with other invocations of do_marking_step() on different tasks ysr@777: (but only one per task, obviously) and concurrently with the ysr@777: mutator threads, or during remark, hence it eliminates the need ysr@777: for two versions of the code. When called during remark, it will ysr@777: pick up from where the task left off during the concurrent marking ysr@777: phase. Interestingly, tasks are also claimable during evacuation ysr@777: pauses too, since do_marking_step() ensures that it aborts before ysr@777: it needs to yield. ysr@777: johnc@4787: The data structures that it uses to do marking work are the ysr@777: following: ysr@777: ysr@777: (1) Marking Bitmap. If there are gray objects that appear only ysr@777: on the bitmap (this happens either when dealing with an overflow ysr@777: or when the initial marking phase has simply marked the roots ysr@777: and didn't push them on the stack), then tasks claim heap ysr@777: regions whose bitmap they then scan to find gray objects. A ysr@777: global finger indicates where the end of the last claimed region ysr@777: is. A local finger indicates how far into the region a task has ysr@777: scanned. The two fingers are used to determine how to gray an ysr@777: object (i.e. whether simply marking it is OK, as it will be ysr@777: visited by a task in the future, or whether it needs to be also ysr@777: pushed on a stack). ysr@777: ysr@777: (2) Local Queue. The local queue of the task which is accessed ysr@777: reasonably efficiently by the task. Other tasks can steal from ysr@777: it when they run out of work. Throughout the marking phase, a ysr@777: task attempts to keep its local queue short but not totally ysr@777: empty, so that entries are available for stealing by other ysr@777: tasks. Only when there is no more work, a task will totally ysr@777: drain its local queue. ysr@777: ysr@777: (3) Global Mark Stack. This handles local queue overflow. During ysr@777: marking only sets of entries are moved between it and the local ysr@777: queues, as access to it requires a mutex and more fine-grain ysr@777: interaction with it which might cause contention. If it ysr@777: overflows, then the marking phase should restart and iterate ysr@777: over the bitmap to identify gray objects. Throughout the marking ysr@777: phase, tasks attempt to keep the global mark stack at a small ysr@777: length but not totally empty, so that entries are available for ysr@777: popping by other tasks. Only when there is no more work, tasks ysr@777: will totally drain the global mark stack. ysr@777: tonyp@3691: (4) SATB Buffer Queue. This is where completed SATB buffers are ysr@777: made available. Buffers are regularly removed from this queue ysr@777: and scanned for roots, so that the queue doesn't get too ysr@777: long. During remark, all completed buffers are processed, as ysr@777: well as the filled in parts of any uncompleted buffers. ysr@777: ysr@777: The do_marking_step() method tries to abort when the time target ysr@777: has been reached. There are a few other cases when the ysr@777: do_marking_step() method also aborts: ysr@777: ysr@777: (1) When the marking phase has been aborted (after a Full GC). ysr@777: tonyp@3691: (2) When a global overflow (on the global stack) has been tonyp@3691: triggered. Before the task aborts, it will actually sync up with tonyp@3691: the other tasks to ensure that all the marking data structures johnc@4788: (local queues, stacks, fingers etc.) are re-initialized so that tonyp@3691: when do_marking_step() completes, the marking phase can tonyp@3691: immediately restart. ysr@777: ysr@777: (3) When enough completed SATB buffers are available. The ysr@777: do_marking_step() method only tries to drain SATB buffers right ysr@777: at the beginning. So, if enough buffers are available, the ysr@777: marking step aborts and the SATB buffers are processed at ysr@777: the beginning of the next invocation. ysr@777: ysr@777: (4) To yield. when we have to yield then we abort and yield ysr@777: right at the end of do_marking_step(). This saves us from a lot ysr@777: of hassle as, by yielding we might allow a Full GC. If this ysr@777: happens then objects will be compacted underneath our feet, the ysr@777: heap might shrink, etc. We save checking for this by just ysr@777: aborting and doing the yield right at the end. ysr@777: ysr@777: From the above it follows that the do_marking_step() method should ysr@777: be called in a loop (or, otherwise, regularly) until it completes. ysr@777: ysr@777: If a marking step completes without its has_aborted() flag being ysr@777: true, it means it has completed the current marking phase (and ysr@777: also all other marking tasks have done so and have all synced up). ysr@777: ysr@777: A method called regular_clock_call() is invoked "regularly" (in ysr@777: sub ms intervals) throughout marking. It is this clock method that ysr@777: checks all the abort conditions which were mentioned above and ysr@777: decides when the task should abort. A work-based scheme is used to ysr@777: trigger this clock method: when the number of object words the ysr@777: marking phase has scanned or the number of references the marking ysr@777: phase has visited reach a given limit. Additional invocations to ysr@777: the method clock have been planted in a few other strategic places ysr@777: too. The initial reason for the clock method was to avoid calling ysr@777: vtime too regularly, as it is quite expensive. So, once it was in ysr@777: place, it was natural to piggy-back all the other conditions on it ysr@777: too and not constantly check them throughout the code. ysr@777: johnc@4787: If do_termination is true then do_marking_step will enter its johnc@4787: termination protocol. johnc@4787: johnc@4787: The value of is_serial must be true when do_marking_step is being johnc@4787: called serially (i.e. by the VMThread) and do_marking_step should johnc@4787: skip any synchronization in the termination and overflow code. johnc@4787: Examples include the serial remark code and the serial reference johnc@4787: processing closures. johnc@4787: johnc@4787: The value of is_serial must be false when do_marking_step is johnc@4787: being called by any of the worker threads in a work gang. johnc@4787: Examples include the concurrent marking code (CMMarkingTask), johnc@4787: the MT remark code, and the MT reference processing closures. johnc@4787: ysr@777: *****************************************************************************/ ysr@777: johnc@2494: void CMTask::do_marking_step(double time_target_ms, johnc@4787: bool do_termination, johnc@4787: bool is_serial) { tonyp@1458: assert(time_target_ms >= 1.0, "minimum granularity is 1ms"); tonyp@1458: assert(concurrent() == _cm->concurrent(), "they should be the same"); tonyp@1458: ysr@777: G1CollectorPolicy* g1_policy = _g1h->g1_policy(); tonyp@1458: assert(_task_queues != NULL, "invariant"); tonyp@1458: assert(_task_queue != NULL, "invariant"); johnc@4173: assert(_task_queues->queue(_worker_id) == _task_queue, "invariant"); tonyp@1458: tonyp@1458: assert(!_claimed, tonyp@1458: "only one thread should claim this task at any one time"); ysr@777: ysr@777: // OK, this doesn't safeguard again all possible scenarios, as it is ysr@777: // possible for two threads to set the _claimed flag at the same ysr@777: // time. But it is only for debugging purposes anyway and it will ysr@777: // catch most problems. ysr@777: _claimed = true; ysr@777: ysr@777: _start_time_ms = os::elapsedVTime() * 1000.0; ysr@777: statsOnly( _interval_start_time_ms = _start_time_ms ); ysr@777: johnc@4787: // If do_stealing is true then do_marking_step will attempt to johnc@4787: // steal work from the other CMTasks. It only makes sense to johnc@4787: // enable stealing when the termination protocol is enabled johnc@4787: // and do_marking_step() is not being called serially. johnc@4787: bool do_stealing = do_termination && !is_serial; johnc@4787: ysr@777: double diff_prediction_ms = ysr@777: g1_policy->get_new_prediction(&_marking_step_diffs_ms); ysr@777: _time_target_ms = time_target_ms - diff_prediction_ms; ysr@777: ysr@777: // set up the variables that are used in the work-based scheme to ysr@777: // call the regular clock method ysr@777: _words_scanned = 0; ysr@777: _refs_reached = 0; ysr@777: recalculate_limits(); ysr@777: ysr@777: // clear all flags ysr@777: clear_has_aborted(); johnc@2494: _has_timed_out = false; ysr@777: _draining_satb_buffers = false; ysr@777: ysr@777: ++_calls; ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] >>>>>>>>>> START, call = %d, " ysr@777: "target = %1.2lfms >>>>>>>>>>", johnc@4173: _worker_id, _calls, _time_target_ms); tonyp@2973: } ysr@777: ysr@777: // Set up the bitmap and oop closures. Anything that uses them is ysr@777: // eventually called from this method, so it is OK to allocate these ysr@777: // statically. ysr@777: CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap); tonyp@2968: G1CMOopClosure cm_oop_closure(_g1h, _cm, this); tonyp@2968: set_cm_oop_closure(&cm_oop_closure); ysr@777: ysr@777: if (_cm->has_overflown()) { tonyp@3691: // This can happen if the mark stack overflows during a GC pause tonyp@3691: // and this task, after a yield point, restarts. We have to abort tonyp@3691: // as we need to get into the overflow protocol which happens tonyp@3691: // right at the end of this task. ysr@777: set_has_aborted(); ysr@777: } ysr@777: ysr@777: // First drain any available SATB buffers. After this, we will not ysr@777: // look at SATB buffers before the next invocation of this method. ysr@777: // If enough completed SATB buffers are queued up, the regular clock ysr@777: // will abort this task so that it restarts. ysr@777: drain_satb_buffers(); ysr@777: // ...then partially drain the local queue and the global stack ysr@777: drain_local_queue(true); ysr@777: drain_global_stack(true); ysr@777: ysr@777: do { ysr@777: if (!has_aborted() && _curr_region != NULL) { ysr@777: // This means that we're already holding on to a region. tonyp@1458: assert(_finger != NULL, "if region is not NULL, then the finger " tonyp@1458: "should not be NULL either"); ysr@777: ysr@777: // We might have restarted this task after an evacuation pause ysr@777: // which might have evacuated the region we're holding on to ysr@777: // underneath our feet. Let's read its limit again to make sure ysr@777: // that we do not iterate over a region of the heap that ysr@777: // contains garbage (update_region_limit() will also move ysr@777: // _finger to the start of the region if it is found empty). ysr@777: update_region_limit(); ysr@777: // We will start from _finger not from the start of the region, ysr@777: // as we might be restarting this task after aborting half-way ysr@777: // through scanning this region. In this case, _finger points to ysr@777: // the address where we last found a marked object. If this is a ysr@777: // fresh region, _finger points to start(). ysr@777: MemRegion mr = MemRegion(_finger, _region_limit); ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] we're scanning part " ysr@777: "["PTR_FORMAT", "PTR_FORMAT") " johnc@4580: "of region "HR_FORMAT, johnc@4580: _worker_id, _finger, _region_limit, johnc@4580: HR_FORMAT_PARAMS(_curr_region)); tonyp@2973: } ysr@777: johnc@4580: assert(!_curr_region->isHumongous() || mr.start() == _curr_region->bottom(), johnc@4580: "humongous regions should go around loop once only"); johnc@4580: johnc@4580: // Some special cases: johnc@4580: // If the memory region is empty, we can just give up the region. johnc@4580: // If the current region is humongous then we only need to check johnc@4580: // the bitmap for the bit associated with the start of the object, johnc@4580: // scan the object if it's live, and give up the region. johnc@4580: // Otherwise, let's iterate over the bitmap of the part of the region johnc@4580: // that is left. johnc@4575: // If the iteration is successful, give up the region. johnc@4580: if (mr.is_empty()) { johnc@4580: giveup_current_region(); johnc@4580: regular_clock_call(); johnc@4580: } else if (_curr_region->isHumongous() && mr.start() == _curr_region->bottom()) { johnc@4580: if (_nextMarkBitMap->isMarked(mr.start())) { johnc@4580: // The object is marked - apply the closure johnc@4580: BitMap::idx_t offset = _nextMarkBitMap->heapWordToOffset(mr.start()); johnc@4580: bitmap_closure.do_bit(offset); johnc@4580: } johnc@4580: // Even if this task aborted while scanning the humongous object johnc@4580: // we can (and should) give up the current region. johnc@4580: giveup_current_region(); johnc@4580: regular_clock_call(); johnc@4580: } else if (_nextMarkBitMap->iterate(&bitmap_closure, mr)) { ysr@777: giveup_current_region(); ysr@777: regular_clock_call(); ysr@777: } else { tonyp@1458: assert(has_aborted(), "currently the only way to do so"); ysr@777: // The only way to abort the bitmap iteration is to return ysr@777: // false from the do_bit() method. However, inside the ysr@777: // do_bit() method we move the _finger to point to the ysr@777: // object currently being looked at. So, if we bail out, we ysr@777: // have definitely set _finger to something non-null. tonyp@1458: assert(_finger != NULL, "invariant"); ysr@777: ysr@777: // Region iteration was actually aborted. So now _finger ysr@777: // points to the address of the object we last scanned. If we ysr@777: // leave it there, when we restart this task, we will rescan ysr@777: // the object. It is easy to avoid this. We move the finger by ysr@777: // enough to point to the next possible object header (the ysr@777: // bitmap knows by how much we need to move it as it knows its ysr@777: // granularity). apetrusenko@1749: assert(_finger < _region_limit, "invariant"); tamao@4733: HeapWord* new_finger = _nextMarkBitMap->nextObject(_finger); apetrusenko@1749: // Check if bitmap iteration was aborted while scanning the last object apetrusenko@1749: if (new_finger >= _region_limit) { tonyp@3691: giveup_current_region(); apetrusenko@1749: } else { tonyp@3691: move_finger_to(new_finger); apetrusenko@1749: } ysr@777: } ysr@777: } ysr@777: // At this point we have either completed iterating over the ysr@777: // region we were holding on to, or we have aborted. ysr@777: ysr@777: // We then partially drain the local queue and the global stack. ysr@777: // (Do we really need this?) ysr@777: drain_local_queue(true); ysr@777: drain_global_stack(true); ysr@777: ysr@777: // Read the note on the claim_region() method on why it might ysr@777: // return NULL with potentially more regions available for ysr@777: // claiming and why we have to check out_of_regions() to determine ysr@777: // whether we're done or not. ysr@777: while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) { ysr@777: // We are going to try to claim a new region. We should have ysr@777: // given up on the previous one. tonyp@1458: // Separated the asserts so that we know which one fires. tonyp@1458: assert(_curr_region == NULL, "invariant"); tonyp@1458: assert(_finger == NULL, "invariant"); tonyp@1458: assert(_region_limit == NULL, "invariant"); tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] trying to claim a new region", _worker_id); tonyp@2973: } johnc@4173: HeapRegion* claimed_region = _cm->claim_region(_worker_id); ysr@777: if (claimed_region != NULL) { ysr@777: // Yes, we managed to claim one ysr@777: statsOnly( ++_regions_claimed ); ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] we successfully claimed " ysr@777: "region "PTR_FORMAT, johnc@4173: _worker_id, claimed_region); tonyp@2973: } ysr@777: ysr@777: setup_for_region(claimed_region); tonyp@1458: assert(_curr_region == claimed_region, "invariant"); ysr@777: } ysr@777: // It is important to call the regular clock here. It might take ysr@777: // a while to claim a region if, for example, we hit a large ysr@777: // block of empty regions. So we need to call the regular clock ysr@777: // method once round the loop to make sure it's called ysr@777: // frequently enough. ysr@777: regular_clock_call(); ysr@777: } ysr@777: ysr@777: if (!has_aborted() && _curr_region == NULL) { tonyp@1458: assert(_cm->out_of_regions(), tonyp@1458: "at this point we should be out of regions"); ysr@777: } ysr@777: } while ( _curr_region != NULL && !has_aborted()); ysr@777: ysr@777: if (!has_aborted()) { ysr@777: // We cannot check whether the global stack is empty, since other tonyp@3691: // tasks might be pushing objects to it concurrently. tonyp@1458: assert(_cm->out_of_regions(), tonyp@1458: "at this point we should be out of regions"); ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] all regions claimed", _worker_id); tonyp@2973: } ysr@777: ysr@777: // Try to reduce the number of available SATB buffers so that ysr@777: // remark has less work to do. ysr@777: drain_satb_buffers(); ysr@777: } ysr@777: ysr@777: // Since we've done everything else, we can now totally drain the ysr@777: // local queue and global stack. ysr@777: drain_local_queue(false); ysr@777: drain_global_stack(false); ysr@777: ysr@777: // Attempt at work stealing from other task's queues. johnc@2494: if (do_stealing && !has_aborted()) { ysr@777: // We have not aborted. This means that we have finished all that ysr@777: // we could. Let's try to do some stealing... ysr@777: ysr@777: // We cannot check whether the global stack is empty, since other tonyp@3691: // tasks might be pushing objects to it concurrently. tonyp@1458: assert(_cm->out_of_regions() && _task_queue->size() == 0, tonyp@1458: "only way to reach here"); ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] starting to steal", _worker_id); tonyp@2973: } ysr@777: ysr@777: while (!has_aborted()) { ysr@777: oop obj; ysr@777: statsOnly( ++_steal_attempts ); ysr@777: johnc@4173: if (_cm->try_stealing(_worker_id, &_hash_seed, obj)) { tonyp@2973: if (_cm->verbose_medium()) { johnc@4173: gclog_or_tty->print_cr("[%u] stolen "PTR_FORMAT" successfully", johnc@4173: _worker_id, (void*) obj); tonyp@2973: } ysr@777: ysr@777: statsOnly( ++_steals ); ysr@777: tonyp@1458: assert(_nextMarkBitMap->isMarked((HeapWord*) obj), tonyp@1458: "any stolen object should be marked"); ysr@777: scan_object(obj); ysr@777: ysr@777: // And since we're towards the end, let's totally drain the ysr@777: // local queue and global stack. ysr@777: drain_local_queue(false); ysr@777: drain_global_stack(false); ysr@777: } else { ysr@777: break; ysr@777: } ysr@777: } ysr@777: } ysr@777: tonyp@2848: // If we are about to wrap up and go into termination, check if we tonyp@2848: // should raise the overflow flag. tonyp@2848: if (do_termination && !has_aborted()) { tonyp@2848: if (_cm->force_overflow()->should_force()) { tonyp@2848: _cm->set_has_overflown(); tonyp@2848: regular_clock_call(); tonyp@2848: } tonyp@2848: } tonyp@2848: ysr@777: // We still haven't aborted. Now, let's try to get into the ysr@777: // termination protocol. johnc@2494: if (do_termination && !has_aborted()) { ysr@777: // We cannot check whether the global stack is empty, since other tonyp@3691: // tasks might be concurrently pushing objects on it. tonyp@1458: // Separated the asserts so that we know which one fires. tonyp@1458: assert(_cm->out_of_regions(), "only way to reach here"); tonyp@1458: assert(_task_queue->size() == 0, "only way to reach here"); ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] starting termination protocol", _worker_id); tonyp@2973: } ysr@777: ysr@777: _termination_start_time_ms = os::elapsedVTime() * 1000.0; johnc@4787: ysr@777: // The CMTask class also extends the TerminatorTerminator class, ysr@777: // hence its should_exit_termination() method will also decide ysr@777: // whether to exit the termination protocol or not. johnc@4787: bool finished = (is_serial || johnc@4787: _cm->terminator()->offer_termination(this)); ysr@777: double termination_end_time_ms = os::elapsedVTime() * 1000.0; ysr@777: _termination_time_ms += ysr@777: termination_end_time_ms - _termination_start_time_ms; ysr@777: ysr@777: if (finished) { ysr@777: // We're all done. ysr@777: johnc@4173: if (_worker_id == 0) { ysr@777: // let's allow task 0 to do this ysr@777: if (concurrent()) { tonyp@1458: assert(_cm->concurrent_marking_in_progress(), "invariant"); ysr@777: // we need to set this to false before the next ysr@777: // safepoint. This way we ensure that the marking phase ysr@777: // doesn't observe any more heap expansions. ysr@777: _cm->clear_concurrent_marking_in_progress(); ysr@777: } ysr@777: } ysr@777: ysr@777: // We can now guarantee that the global stack is empty, since tonyp@1458: // all other tasks have finished. We separated the guarantees so tonyp@1458: // that, if a condition is false, we can immediately find out tonyp@1458: // which one. tonyp@1458: guarantee(_cm->out_of_regions(), "only way to reach here"); tonyp@1458: guarantee(_cm->mark_stack_empty(), "only way to reach here"); tonyp@1458: guarantee(_task_queue->size() == 0, "only way to reach here"); tonyp@1458: guarantee(!_cm->has_overflown(), "only way to reach here"); tonyp@1458: guarantee(!_cm->mark_stack_overflow(), "only way to reach here"); ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] all tasks terminated", _worker_id); tonyp@2973: } ysr@777: } else { ysr@777: // Apparently there's more work to do. Let's abort this task. It ysr@777: // will restart it and we can hopefully find more things to do. ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] apparently there is more work to do", johnc@4173: _worker_id); tonyp@2973: } ysr@777: ysr@777: set_has_aborted(); ysr@777: statsOnly( ++_aborted_termination ); ysr@777: } ysr@777: } ysr@777: ysr@777: // Mainly for debugging purposes to make sure that a pointer to the ysr@777: // closure which was statically allocated in this frame doesn't ysr@777: // escape it by accident. tonyp@2968: set_cm_oop_closure(NULL); ysr@777: double end_time_ms = os::elapsedVTime() * 1000.0; ysr@777: double elapsed_time_ms = end_time_ms - _start_time_ms; ysr@777: // Update the step history. ysr@777: _step_times_ms.add(elapsed_time_ms); ysr@777: ysr@777: if (has_aborted()) { ysr@777: // The task was aborted for some reason. ysr@777: ysr@777: statsOnly( ++_aborted ); ysr@777: johnc@2494: if (_has_timed_out) { ysr@777: double diff_ms = elapsed_time_ms - _time_target_ms; ysr@777: // Keep statistics of how well we did with respect to hitting ysr@777: // our target only if we actually timed out (if we aborted for ysr@777: // other reasons, then the results might get skewed). ysr@777: _marking_step_diffs_ms.add(diff_ms); ysr@777: } ysr@777: ysr@777: if (_cm->has_overflown()) { ysr@777: // This is the interesting one. We aborted because a global ysr@777: // overflow was raised. This means we have to restart the ysr@777: // marking phase and start iterating over regions. However, in ysr@777: // order to do this we have to make sure that all tasks stop ysr@777: // what they are doing and re-initialise in a safe manner. We ysr@777: // will achieve this with the use of two barrier sync points. ysr@777: tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] detected overflow", _worker_id); tonyp@2973: } ysr@777: johnc@4787: if (!is_serial) { johnc@4787: // We only need to enter the sync barrier if being called johnc@4787: // from a parallel context johnc@4787: _cm->enter_first_sync_barrier(_worker_id); johnc@4787: johnc@4787: // When we exit this sync barrier we know that all tasks have johnc@4787: // stopped doing marking work. So, it's now safe to johnc@4787: // re-initialise our data structures. At the end of this method, johnc@4787: // task 0 will clear the global data structures. johnc@4787: } ysr@777: ysr@777: statsOnly( ++_aborted_overflow ); ysr@777: ysr@777: // We clear the local state of this task... ysr@777: clear_region_fields(); ysr@777: johnc@4787: if (!is_serial) { johnc@4787: // ...and enter the second barrier. johnc@4787: _cm->enter_second_sync_barrier(_worker_id); johnc@4787: } johnc@4788: // At this point, if we're during the concurrent phase of johnc@4788: // marking, everything has been re-initialized and we're ysr@777: // ready to restart. ysr@777: } ysr@777: ysr@777: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] <<<<<<<<<< ABORTING, target = %1.2lfms, " ysr@777: "elapsed = %1.2lfms <<<<<<<<<<", johnc@4173: _worker_id, _time_target_ms, elapsed_time_ms); tonyp@2973: if (_cm->has_aborted()) { johnc@4173: gclog_or_tty->print_cr("[%u] ========== MARKING ABORTED ==========", johnc@4173: _worker_id); tonyp@2973: } ysr@777: } ysr@777: } else { tonyp@2973: if (_cm->verbose_low()) { johnc@4173: gclog_or_tty->print_cr("[%u] <<<<<<<<<< FINISHED, target = %1.2lfms, " ysr@777: "elapsed = %1.2lfms <<<<<<<<<<", johnc@4173: _worker_id, _time_target_ms, elapsed_time_ms); tonyp@2973: } ysr@777: } ysr@777: ysr@777: _claimed = false; ysr@777: } ysr@777: johnc@4173: CMTask::CMTask(uint worker_id, ysr@777: ConcurrentMark* cm, johnc@3463: size_t* marked_bytes, johnc@3463: BitMap* card_bm, ysr@777: CMTaskQueue* task_queue, ysr@777: CMTaskQueueSet* task_queues) ysr@777: : _g1h(G1CollectedHeap::heap()), johnc@4173: _worker_id(worker_id), _cm(cm), ysr@777: _claimed(false), ysr@777: _nextMarkBitMap(NULL), _hash_seed(17), ysr@777: _task_queue(task_queue), ysr@777: _task_queues(task_queues), tonyp@2968: _cm_oop_closure(NULL), johnc@3463: _marked_bytes_array(marked_bytes), johnc@3463: _card_bm(card_bm) { tonyp@1458: guarantee(task_queue != NULL, "invariant"); tonyp@1458: guarantee(task_queues != NULL, "invariant"); ysr@777: ysr@777: statsOnly( _clock_due_to_scanning = 0; ysr@777: _clock_due_to_marking = 0 ); ysr@777: ysr@777: _marking_step_diffs_ms.add(0.5); ysr@777: } tonyp@2717: tonyp@2717: // These are formatting macros that are used below to ensure tonyp@2717: // consistent formatting. The *_H_* versions are used to format the tonyp@2717: // header for a particular value and they should be kept consistent tonyp@2717: // with the corresponding macro. Also note that most of the macros add tonyp@2717: // the necessary white space (as a prefix) which makes them a bit tonyp@2717: // easier to compose. tonyp@2717: tonyp@2717: // All the output lines are prefixed with this string to be able to tonyp@2717: // identify them easily in a large log file. tonyp@2717: #define G1PPRL_LINE_PREFIX "###" tonyp@2717: tonyp@2717: #define G1PPRL_ADDR_BASE_FORMAT " "PTR_FORMAT"-"PTR_FORMAT tonyp@2717: #ifdef _LP64 tonyp@2717: #define G1PPRL_ADDR_BASE_H_FORMAT " %37s" tonyp@2717: #else // _LP64 tonyp@2717: #define G1PPRL_ADDR_BASE_H_FORMAT " %21s" tonyp@2717: #endif // _LP64 tonyp@2717: tonyp@2717: // For per-region info tonyp@2717: #define G1PPRL_TYPE_FORMAT " %-4s" tonyp@2717: #define G1PPRL_TYPE_H_FORMAT " %4s" tonyp@2717: #define G1PPRL_BYTE_FORMAT " "SIZE_FORMAT_W(9) tonyp@2717: #define G1PPRL_BYTE_H_FORMAT " %9s" tonyp@2717: #define G1PPRL_DOUBLE_FORMAT " %14.1f" tonyp@2717: #define G1PPRL_DOUBLE_H_FORMAT " %14s" tonyp@2717: tonyp@2717: // For summary info tonyp@2717: #define G1PPRL_SUM_ADDR_FORMAT(tag) " "tag":"G1PPRL_ADDR_BASE_FORMAT tonyp@2717: #define G1PPRL_SUM_BYTE_FORMAT(tag) " "tag": "SIZE_FORMAT tonyp@2717: #define G1PPRL_SUM_MB_FORMAT(tag) " "tag": %1.2f MB" tonyp@2717: #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag)" / %1.2f %%" tonyp@2717: tonyp@2717: G1PrintRegionLivenessInfoClosure:: tonyp@2717: G1PrintRegionLivenessInfoClosure(outputStream* out, const char* phase_name) tonyp@2717: : _out(out), tonyp@2717: _total_used_bytes(0), _total_capacity_bytes(0), tonyp@2717: _total_prev_live_bytes(0), _total_next_live_bytes(0), tonyp@2717: _hum_used_bytes(0), _hum_capacity_bytes(0), tschatzl@5122: _hum_prev_live_bytes(0), _hum_next_live_bytes(0), johnc@5548: _total_remset_bytes(0), _total_strong_code_roots_bytes(0) { tonyp@2717: G1CollectedHeap* g1h = G1CollectedHeap::heap(); tonyp@2717: MemRegion g1_committed = g1h->g1_committed(); tonyp@2717: MemRegion g1_reserved = g1h->g1_reserved(); tonyp@2717: double now = os::elapsedTime(); tonyp@2717: tonyp@2717: // Print the header of the output. tonyp@2717: _out->cr(); tonyp@2717: _out->print_cr(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now); tonyp@2717: _out->print_cr(G1PPRL_LINE_PREFIX" HEAP" tonyp@2717: G1PPRL_SUM_ADDR_FORMAT("committed") tonyp@2717: G1PPRL_SUM_ADDR_FORMAT("reserved") tonyp@2717: G1PPRL_SUM_BYTE_FORMAT("region-size"), tonyp@2717: g1_committed.start(), g1_committed.end(), tonyp@2717: g1_reserved.start(), g1_reserved.end(), johnc@3182: HeapRegion::GrainBytes); tonyp@2717: _out->print_cr(G1PPRL_LINE_PREFIX); tonyp@2717: _out->print_cr(G1PPRL_LINE_PREFIX tschatzl@5122: G1PPRL_TYPE_H_FORMAT tschatzl@5122: G1PPRL_ADDR_BASE_H_FORMAT tschatzl@5122: G1PPRL_BYTE_H_FORMAT tschatzl@5122: G1PPRL_BYTE_H_FORMAT tschatzl@5122: G1PPRL_BYTE_H_FORMAT tschatzl@5122: G1PPRL_DOUBLE_H_FORMAT johnc@5548: G1PPRL_BYTE_H_FORMAT tschatzl@5122: G1PPRL_BYTE_H_FORMAT, tschatzl@5122: "type", "address-range", johnc@5548: "used", "prev-live", "next-live", "gc-eff", johnc@5548: "remset", "code-roots"); johnc@3173: _out->print_cr(G1PPRL_LINE_PREFIX tschatzl@5122: G1PPRL_TYPE_H_FORMAT tschatzl@5122: G1PPRL_ADDR_BASE_H_FORMAT tschatzl@5122: G1PPRL_BYTE_H_FORMAT tschatzl@5122: G1PPRL_BYTE_H_FORMAT tschatzl@5122: G1PPRL_BYTE_H_FORMAT tschatzl@5122: G1PPRL_DOUBLE_H_FORMAT johnc@5548: G1PPRL_BYTE_H_FORMAT tschatzl@5122: G1PPRL_BYTE_H_FORMAT, tschatzl@5122: "", "", johnc@5548: "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)", johnc@5548: "(bytes)", "(bytes)"); tonyp@2717: } tonyp@2717: tonyp@2717: // It takes as a parameter a reference to one of the _hum_* fields, it tonyp@2717: // deduces the corresponding value for a region in a humongous region tonyp@2717: // series (either the region size, or what's left if the _hum_* field tonyp@2717: // is < the region size), and updates the _hum_* field accordingly. tonyp@2717: size_t G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* hum_bytes) { tonyp@2717: size_t bytes = 0; tonyp@2717: // The > 0 check is to deal with the prev and next live bytes which tonyp@2717: // could be 0. tonyp@2717: if (*hum_bytes > 0) { johnc@3182: bytes = MIN2(HeapRegion::GrainBytes, *hum_bytes); tonyp@2717: *hum_bytes -= bytes; tonyp@2717: } tonyp@2717: return bytes; tonyp@2717: } tonyp@2717: tonyp@2717: // It deduces the values for a region in a humongous region series tonyp@2717: // from the _hum_* fields and updates those accordingly. It assumes tonyp@2717: // that that _hum_* fields have already been set up from the "starts tonyp@2717: // humongous" region and we visit the regions in address order. tonyp@2717: void G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* used_bytes, tonyp@2717: size_t* capacity_bytes, tonyp@2717: size_t* prev_live_bytes, tonyp@2717: size_t* next_live_bytes) { tonyp@2717: assert(_hum_used_bytes > 0 && _hum_capacity_bytes > 0, "pre-condition"); tonyp@2717: *used_bytes = get_hum_bytes(&_hum_used_bytes); tonyp@2717: *capacity_bytes = get_hum_bytes(&_hum_capacity_bytes); tonyp@2717: *prev_live_bytes = get_hum_bytes(&_hum_prev_live_bytes); tonyp@2717: *next_live_bytes = get_hum_bytes(&_hum_next_live_bytes); tonyp@2717: } tonyp@2717: tonyp@2717: bool G1PrintRegionLivenessInfoClosure::doHeapRegion(HeapRegion* r) { tonyp@2717: const char* type = ""; tonyp@2717: HeapWord* bottom = r->bottom(); tonyp@2717: HeapWord* end = r->end(); tonyp@2717: size_t capacity_bytes = r->capacity(); tonyp@2717: size_t used_bytes = r->used(); tonyp@2717: size_t prev_live_bytes = r->live_bytes(); tonyp@2717: size_t next_live_bytes = r->next_live_bytes(); tonyp@2717: double gc_eff = r->gc_efficiency(); tschatzl@5122: size_t remset_bytes = r->rem_set()->mem_size(); johnc@5548: size_t strong_code_roots_bytes = r->rem_set()->strong_code_roots_mem_size(); johnc@5548: tonyp@2717: if (r->used() == 0) { tonyp@2717: type = "FREE"; tonyp@2717: } else if (r->is_survivor()) { tonyp@2717: type = "SURV"; tonyp@2717: } else if (r->is_young()) { tonyp@2717: type = "EDEN"; tonyp@2717: } else if (r->startsHumongous()) { tonyp@2717: type = "HUMS"; tonyp@2717: tonyp@2717: assert(_hum_used_bytes == 0 && _hum_capacity_bytes == 0 && tonyp@2717: _hum_prev_live_bytes == 0 && _hum_next_live_bytes == 0, tonyp@2717: "they should have been zeroed after the last time we used them"); tonyp@2717: // Set up the _hum_* fields. tonyp@2717: _hum_capacity_bytes = capacity_bytes; tonyp@2717: _hum_used_bytes = used_bytes; tonyp@2717: _hum_prev_live_bytes = prev_live_bytes; tonyp@2717: _hum_next_live_bytes = next_live_bytes; tonyp@2717: get_hum_bytes(&used_bytes, &capacity_bytes, tonyp@2717: &prev_live_bytes, &next_live_bytes); tonyp@2717: end = bottom + HeapRegion::GrainWords; tonyp@2717: } else if (r->continuesHumongous()) { tonyp@2717: type = "HUMC"; tonyp@2717: get_hum_bytes(&used_bytes, &capacity_bytes, tonyp@2717: &prev_live_bytes, &next_live_bytes); tonyp@2717: assert(end == bottom + HeapRegion::GrainWords, "invariant"); tonyp@2717: } else { tonyp@2717: type = "OLD"; tonyp@2717: } tonyp@2717: tonyp@2717: _total_used_bytes += used_bytes; tonyp@2717: _total_capacity_bytes += capacity_bytes; tonyp@2717: _total_prev_live_bytes += prev_live_bytes; tonyp@2717: _total_next_live_bytes += next_live_bytes; tschatzl@5122: _total_remset_bytes += remset_bytes; johnc@5548: _total_strong_code_roots_bytes += strong_code_roots_bytes; tonyp@2717: tonyp@2717: // Print a line for this particular region. tonyp@2717: _out->print_cr(G1PPRL_LINE_PREFIX tonyp@2717: G1PPRL_TYPE_FORMAT tonyp@2717: G1PPRL_ADDR_BASE_FORMAT tonyp@2717: G1PPRL_BYTE_FORMAT tonyp@2717: G1PPRL_BYTE_FORMAT tonyp@2717: G1PPRL_BYTE_FORMAT tschatzl@5122: G1PPRL_DOUBLE_FORMAT johnc@5548: G1PPRL_BYTE_FORMAT tschatzl@5122: G1PPRL_BYTE_FORMAT, tonyp@2717: type, bottom, end, johnc@5548: used_bytes, prev_live_bytes, next_live_bytes, gc_eff, johnc@5548: remset_bytes, strong_code_roots_bytes); tonyp@2717: tonyp@2717: return false; tonyp@2717: } tonyp@2717: tonyp@2717: G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() { tschatzl@5122: // add static memory usages to remembered set sizes tschatzl@5122: _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size(); tonyp@2717: // Print the footer of the output. tonyp@2717: _out->print_cr(G1PPRL_LINE_PREFIX); tonyp@2717: _out->print_cr(G1PPRL_LINE_PREFIX tonyp@2717: " SUMMARY" tonyp@2717: G1PPRL_SUM_MB_FORMAT("capacity") tonyp@2717: G1PPRL_SUM_MB_PERC_FORMAT("used") tonyp@2717: G1PPRL_SUM_MB_PERC_FORMAT("prev-live") tschatzl@5122: G1PPRL_SUM_MB_PERC_FORMAT("next-live") johnc@5548: G1PPRL_SUM_MB_FORMAT("remset") johnc@5548: G1PPRL_SUM_MB_FORMAT("code-roots"), tonyp@2717: bytes_to_mb(_total_capacity_bytes), tonyp@2717: bytes_to_mb(_total_used_bytes), tonyp@2717: perc(_total_used_bytes, _total_capacity_bytes), tonyp@2717: bytes_to_mb(_total_prev_live_bytes), tonyp@2717: perc(_total_prev_live_bytes, _total_capacity_bytes), tonyp@2717: bytes_to_mb(_total_next_live_bytes), tschatzl@5122: perc(_total_next_live_bytes, _total_capacity_bytes), johnc@5548: bytes_to_mb(_total_remset_bytes), johnc@5548: bytes_to_mb(_total_strong_code_roots_bytes)); tonyp@2717: _out->cr(); tonyp@2717: }