1.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp Tue Jan 24 17:08:58 2012 -0500
1.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp Thu Jan 12 00:06:47 2012 -0800
1.3 @@ -28,6 +28,159 @@
1.4 #include "gc_implementation/g1/concurrentMark.hpp"
1.5 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
1.6
1.7 +// Returns the index in the liveness accounting card bitmap
1.8 +// for the given address
1.9 +inline BitMap::idx_t ConcurrentMark::card_bitmap_index_for(HeapWord* addr) {
1.10 + // Below, the term "card num" means the result of shifting an address
1.11 + // by the card shift -- address 0 corresponds to card number 0. One
1.12 + // must subtract the card num of the bottom of the heap to obtain a
1.13 + // card table index.
1.14 +
1.15 + intptr_t card_num = intptr_t(uintptr_t(addr) >> CardTableModRefBS::card_shift);
1.16 + return card_num - heap_bottom_card_num();
1.17 +}
1.18 +
1.19 +// Counts the given memory region in the given task/worker
1.20 +// counting data structures.
1.21 +inline void ConcurrentMark::count_region(MemRegion mr, HeapRegion* hr,
1.22 + size_t* marked_bytes_array,
1.23 + BitMap* task_card_bm) {
1.24 + G1CollectedHeap* g1h = _g1h;
1.25 + HeapWord* start = mr.start();
1.26 + HeapWord* last = mr.last();
1.27 + size_t region_size_bytes = mr.byte_size();
1.28 + size_t index = hr->hrs_index();
1.29 +
1.30 + assert(!hr->continuesHumongous(), "should not be HC region");
1.31 + assert(hr == g1h->heap_region_containing(start), "sanity");
1.32 + assert(hr == g1h->heap_region_containing(mr.last()), "sanity");
1.33 + assert(marked_bytes_array != NULL, "pre-condition");
1.34 + assert(task_card_bm != NULL, "pre-condition");
1.35 +
1.36 + // Add to the task local marked bytes for this region.
1.37 + marked_bytes_array[index] += region_size_bytes;
1.38 +
1.39 + BitMap::idx_t start_idx = card_bitmap_index_for(start);
1.40 + BitMap::idx_t last_idx = card_bitmap_index_for(last);
1.41 +
1.42 + // The card bitmap is task/worker specific => no need to use 'par' routines.
1.43 + // Set bits in the inclusive bit range [start_idx, last_idx].
1.44 + //
1.45 + // For small ranges use a simple loop; otherwise use set_range
1.46 + // The range are the cards that are spanned by the object/region
1.47 + // so 8 cards will allow objects/regions up to 4K to be handled
1.48 + // using the loop.
1.49 + if ((last_idx - start_idx) <= 8) {
1.50 + for (BitMap::idx_t i = start_idx; i <= last_idx; i += 1) {
1.51 + task_card_bm->set_bit(i);
1.52 + }
1.53 + } else {
1.54 + assert(last_idx < task_card_bm->size(), "sanity");
1.55 + // Note: BitMap::set_range() is exclusive.
1.56 + task_card_bm->set_range(start_idx, last_idx+1);
1.57 + }
1.58 +}
1.59 +
1.60 +// Counts the given memory region, which may be a single object, in the
1.61 +// task/worker counting data structures for the given worker id.
1.62 +inline void ConcurrentMark::count_region(MemRegion mr, uint worker_id) {
1.63 + size_t* marked_bytes_array = count_marked_bytes_array_for(worker_id);
1.64 + BitMap* task_card_bm = count_card_bitmap_for(worker_id);
1.65 + HeapWord* addr = mr.start();
1.66 + HeapRegion* hr = _g1h->heap_region_containing_raw(addr);
1.67 + count_region(mr, hr, marked_bytes_array, task_card_bm);
1.68 +}
1.69 +
1.70 +// Counts the given object in the given task/worker counting data structures.
1.71 +inline void ConcurrentMark::count_object(oop obj,
1.72 + HeapRegion* hr,
1.73 + size_t* marked_bytes_array,
1.74 + BitMap* task_card_bm) {
1.75 + MemRegion mr((HeapWord*)obj, obj->size());
1.76 + count_region(mr, hr, marked_bytes_array, task_card_bm);
1.77 +}
1.78 +
1.79 +// Counts the given object in the task/worker counting data
1.80 +// structures for the given worker id.
1.81 +inline void ConcurrentMark::count_object(oop obj, HeapRegion* hr, uint worker_id) {
1.82 + size_t* marked_bytes_array = count_marked_bytes_array_for(worker_id);
1.83 + BitMap* task_card_bm = count_card_bitmap_for(worker_id);
1.84 + HeapWord* addr = (HeapWord*) obj;
1.85 + count_object(obj, hr, marked_bytes_array, task_card_bm);
1.86 +}
1.87 +
1.88 +// Attempts to mark the given object and, if successful, counts
1.89 +// the object in the given task/worker counting structures.
1.90 +inline bool ConcurrentMark::par_mark_and_count(oop obj,
1.91 + HeapRegion* hr,
1.92 + size_t* marked_bytes_array,
1.93 + BitMap* task_card_bm) {
1.94 + HeapWord* addr = (HeapWord*)obj;
1.95 + if (_nextMarkBitMap->parMark(addr)) {
1.96 + // Update the task specific count data for the object.
1.97 + count_object(obj, hr, marked_bytes_array, task_card_bm);
1.98 + return true;
1.99 + }
1.100 + return false;
1.101 +}
1.102 +
1.103 +// Attempts to mark the given object and, if successful, counts
1.104 +// the object in the task/worker counting structures for the
1.105 +// given worker id.
1.106 +inline bool ConcurrentMark::par_mark_and_count(oop obj,
1.107 + HeapRegion* hr,
1.108 + uint worker_id) {
1.109 + HeapWord* addr = (HeapWord*)obj;
1.110 + if (_nextMarkBitMap->parMark(addr)) {
1.111 + // Update the task specific count data for the object.
1.112 + count_object(obj, hr, worker_id);
1.113 + return true;
1.114 + }
1.115 + return false;
1.116 +}
1.117 +
1.118 +// As above - but we don't know the heap region containing the
1.119 +// object and so have to supply it.
1.120 +inline bool ConcurrentMark::par_mark_and_count(oop obj, uint worker_id) {
1.121 + HeapWord* addr = (HeapWord*)obj;
1.122 + HeapRegion* hr = _g1h->heap_region_containing_raw(addr);
1.123 + return par_mark_and_count(obj, hr, worker_id);
1.124 +}
1.125 +
1.126 +// Similar to the above routine but we already know the size, in words, of
1.127 +// the object that we wish to mark/count
1.128 +inline bool ConcurrentMark::par_mark_and_count(oop obj,
1.129 + size_t word_size,
1.130 + uint worker_id) {
1.131 + HeapWord* addr = (HeapWord*)obj;
1.132 + if (_nextMarkBitMap->parMark(addr)) {
1.133 + // Update the task specific count data for the object.
1.134 + MemRegion mr(addr, word_size);
1.135 + count_region(mr, worker_id);
1.136 + return true;
1.137 + }
1.138 + return false;
1.139 +}
1.140 +
1.141 +// Unconditionally mark the given object, and unconditinally count
1.142 +// the object in the counting structures for worker id 0.
1.143 +// Should *not* be called from parallel code.
1.144 +inline bool ConcurrentMark::mark_and_count(oop obj, HeapRegion* hr) {
1.145 + HeapWord* addr = (HeapWord*)obj;
1.146 + _nextMarkBitMap->mark(addr);
1.147 + // Update the task specific count data for the object.
1.148 + count_object(obj, hr, 0 /* worker_id */);
1.149 + return true;
1.150 +}
1.151 +
1.152 +// As above - but we don't have the heap region containing the
1.153 +// object, so we have to supply it.
1.154 +inline bool ConcurrentMark::mark_and_count(oop obj) {
1.155 + HeapWord* addr = (HeapWord*)obj;
1.156 + HeapRegion* hr = _g1h->heap_region_containing_raw(addr);
1.157 + return mark_and_count(obj, hr);
1.158 +}
1.159 +
1.160 inline bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) {
1.161 HeapWord* start_addr = MAX2(startWord(), mr.start());
1.162 HeapWord* end_addr = MIN2(endWord(), mr.end());
1.163 @@ -113,7 +266,7 @@
1.164
1.165 HeapWord* objAddr = (HeapWord*) obj;
1.166 assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
1.167 - if (_g1h->is_in_g1_reserved(objAddr)) {
1.168 + if (_g1h->is_in_g1_reserved(objAddr)) {
1.169 assert(obj != NULL, "null check is implicit");
1.170 if (!_nextMarkBitMap->isMarked(objAddr)) {
1.171 // Only get the containing region if the object is not marked on the
1.172 @@ -127,9 +280,9 @@
1.173 }
1.174
1.175 // we need to mark it first
1.176 - if (_nextMarkBitMap->parMark(objAddr)) {
1.177 + if (_cm->par_mark_and_count(obj, hr, _marked_bytes_array, _card_bm)) {
1.178 // No OrderAccess:store_load() is needed. It is implicit in the
1.179 - // CAS done in parMark(objAddr) above
1.180 + // CAS done in CMBitMap::parMark() call in the routine above.
1.181 HeapWord* global_finger = _cm->finger();
1.182
1.183 #if _CHECK_BOTH_FINGERS_
1.184 @@ -189,12 +342,7 @@
1.185 ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*) p);
1.186 }
1.187
1.188 -inline void ConcurrentMark::markNext(oop p) {
1.189 - assert(!_nextMarkBitMap->isMarked((HeapWord*) p), "sanity");
1.190 - _nextMarkBitMap->mark((HeapWord*) p);
1.191 -}
1.192 -
1.193 -inline void ConcurrentMark::grayRoot(oop obj, size_t word_size) {
1.194 +inline void ConcurrentMark::grayRoot(oop obj, size_t word_size, uint worker_id) {
1.195 HeapWord* addr = (HeapWord*) obj;
1.196
1.197 // Currently we don't do anything with word_size but we will use it
1.198 @@ -220,7 +368,7 @@
1.199 #endif // ASSERT
1.200
1.201 if (!_nextMarkBitMap->isMarked(addr)) {
1.202 - _nextMarkBitMap->parMark(addr);
1.203 + par_mark_and_count(obj, word_size, worker_id);
1.204 }
1.205 }
1.206
