Wed, 02 Nov 2011 08:04:23 +0100
7106751: G1: gc/gctests/nativeGC03 crashes VM with SIGSEGV
Summary: _cset_rs_update_cl[] was indexed with values beyond what it is set up to handle.
Reviewed-by: ysr, jmasa, johnc
ysr@777 | 1 | /* |
johnc@2504 | 2 | * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. |
ysr@777 | 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
ysr@777 | 4 | * |
ysr@777 | 5 | * This code is free software; you can redistribute it and/or modify it |
ysr@777 | 6 | * under the terms of the GNU General Public License version 2 only, as |
ysr@777 | 7 | * published by the Free Software Foundation. |
ysr@777 | 8 | * |
ysr@777 | 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
ysr@777 | 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
ysr@777 | 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
ysr@777 | 12 | * version 2 for more details (a copy is included in the LICENSE file that |
ysr@777 | 13 | * accompanied this code). |
ysr@777 | 14 | * |
ysr@777 | 15 | * You should have received a copy of the GNU General Public License version |
ysr@777 | 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
ysr@777 | 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
ysr@777 | 18 | * |
trims@1907 | 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
trims@1907 | 20 | * or visit www.oracle.com if you need additional information or have any |
trims@1907 | 21 | * questions. |
ysr@777 | 22 | * |
ysr@777 | 23 | */ |
ysr@777 | 24 | |
stefank@2314 | 25 | #include "precompiled.hpp" |
stefank@2314 | 26 | #include "gc_implementation/g1/concurrentG1Refine.hpp" |
stefank@2314 | 27 | #include "gc_implementation/g1/concurrentG1RefineThread.hpp" |
stefank@2314 | 28 | #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" |
stefank@2314 | 29 | #include "gc_implementation/g1/g1CollectorPolicy.hpp" |
stefank@2314 | 30 | #include "gc_implementation/g1/g1RemSet.hpp" |
stefank@2314 | 31 | #include "gc_implementation/g1/heapRegionSeq.inline.hpp" |
stefank@2314 | 32 | #include "memory/space.inline.hpp" |
stefank@2314 | 33 | #include "runtime/atomic.hpp" |
johnc@2713 | 34 | #include "runtime/java.hpp" |
stefank@2314 | 35 | #include "utilities/copy.hpp" |
ysr@777 | 36 | |
johnc@1325 | 37 | // Possible sizes for the card counts cache: odd primes that roughly double in size. |
johnc@1325 | 38 | // (See jvmtiTagMap.cpp). |
johnc@2713 | 39 | |
johnc@2713 | 40 | #define MAX_SIZE ((size_t) -1) |
johnc@2713 | 41 | |
johnc@2713 | 42 | size_t ConcurrentG1Refine::_cc_cache_sizes[] = { |
johnc@2713 | 43 | 16381, 32771, 76831, 150001, 307261, |
johnc@2713 | 44 | 614563, 1228891, 2457733, 4915219, 9830479, |
johnc@2713 | 45 | 19660831, 39321619, 78643219, 157286461, MAX_SIZE |
johnc@1325 | 46 | }; |
johnc@1325 | 47 | |
ysr@777 | 48 | ConcurrentG1Refine::ConcurrentG1Refine() : |
johnc@1325 | 49 | _card_counts(NULL), _card_epochs(NULL), |
johnc@2713 | 50 | _n_card_counts(0), _max_cards(0), _max_n_card_counts(0), |
johnc@1325 | 51 | _cache_size_index(0), _expand_card_counts(false), |
ysr@777 | 52 | _hot_cache(NULL), |
ysr@777 | 53 | _def_use_cache(false), _use_cache(false), |
tonyp@2716 | 54 | // We initialize the epochs of the array to 0. By initializing |
tonyp@2716 | 55 | // _n_periods to 1 and not 0 we automatically invalidate all the |
tonyp@2716 | 56 | // entries on the array. Otherwise we might accidentally think that |
tonyp@2716 | 57 | // we claimed a card that was in fact never set (see CR7033292). |
tonyp@2716 | 58 | _n_periods(1), |
iveresov@1229 | 59 | _threads(NULL), _n_threads(0) |
ysr@777 | 60 | { |
iveresov@1546 | 61 | |
iveresov@1546 | 62 | // Ergomonically select initial concurrent refinement parameters |
tonyp@1717 | 63 | if (FLAG_IS_DEFAULT(G1ConcRefinementGreenZone)) { |
tonyp@1717 | 64 | FLAG_SET_DEFAULT(G1ConcRefinementGreenZone, MAX2<int>(ParallelGCThreads, 1)); |
iveresov@1546 | 65 | } |
tonyp@1717 | 66 | set_green_zone(G1ConcRefinementGreenZone); |
iveresov@1546 | 67 | |
tonyp@1717 | 68 | if (FLAG_IS_DEFAULT(G1ConcRefinementYellowZone)) { |
tonyp@1717 | 69 | FLAG_SET_DEFAULT(G1ConcRefinementYellowZone, green_zone() * 3); |
iveresov@1546 | 70 | } |
tonyp@1717 | 71 | set_yellow_zone(MAX2<int>(G1ConcRefinementYellowZone, green_zone())); |
iveresov@1546 | 72 | |
tonyp@1717 | 73 | if (FLAG_IS_DEFAULT(G1ConcRefinementRedZone)) { |
tonyp@1717 | 74 | FLAG_SET_DEFAULT(G1ConcRefinementRedZone, yellow_zone() * 2); |
iveresov@1546 | 75 | } |
tonyp@1717 | 76 | set_red_zone(MAX2<int>(G1ConcRefinementRedZone, yellow_zone())); |
iveresov@1546 | 77 | _n_worker_threads = thread_num(); |
iveresov@1546 | 78 | // We need one extra thread to do the young gen rset size sampling. |
iveresov@1546 | 79 | _n_threads = _n_worker_threads + 1; |
iveresov@1546 | 80 | reset_threshold_step(); |
iveresov@1546 | 81 | |
iveresov@1546 | 82 | _threads = NEW_C_HEAP_ARRAY(ConcurrentG1RefineThread*, _n_threads); |
iveresov@1546 | 83 | int worker_id_offset = (int)DirtyCardQueueSet::num_par_ids(); |
iveresov@1546 | 84 | ConcurrentG1RefineThread *next = NULL; |
iveresov@1546 | 85 | for (int i = _n_threads - 1; i >= 0; i--) { |
iveresov@1546 | 86 | ConcurrentG1RefineThread* t = new ConcurrentG1RefineThread(this, next, worker_id_offset, i); |
iveresov@1546 | 87 | assert(t != NULL, "Conc refine should have been created"); |
iveresov@1546 | 88 | assert(t->cg1r() == this, "Conc refine thread should refer to this"); |
iveresov@1546 | 89 | _threads[i] = t; |
iveresov@1546 | 90 | next = t; |
ysr@777 | 91 | } |
ysr@777 | 92 | } |
ysr@777 | 93 | |
iveresov@1546 | 94 | void ConcurrentG1Refine::reset_threshold_step() { |
tonyp@1717 | 95 | if (FLAG_IS_DEFAULT(G1ConcRefinementThresholdStep)) { |
iveresov@1546 | 96 | _thread_threshold_step = (yellow_zone() - green_zone()) / (worker_thread_num() + 1); |
iveresov@1546 | 97 | } else { |
tonyp@1717 | 98 | _thread_threshold_step = G1ConcRefinementThresholdStep; |
iveresov@1230 | 99 | } |
iveresov@1546 | 100 | } |
iveresov@1546 | 101 | |
iveresov@1546 | 102 | int ConcurrentG1Refine::thread_num() { |
tonyp@1717 | 103 | return MAX2<int>((G1ConcRefinementThreads > 0) ? G1ConcRefinementThreads : ParallelGCThreads, 1); |
iveresov@1230 | 104 | } |
iveresov@1230 | 105 | |
ysr@777 | 106 | void ConcurrentG1Refine::init() { |
johnc@1325 | 107 | if (G1ConcRSLogCacheSize > 0) { |
johnc@1325 | 108 | _g1h = G1CollectedHeap::heap(); |
johnc@2713 | 109 | |
johnc@2713 | 110 | _max_cards = _g1h->max_capacity() >> CardTableModRefBS::card_shift; |
johnc@2713 | 111 | _max_n_card_counts = _max_cards * G1MaxHotCardCountSizePercent / 100; |
johnc@1325 | 112 | |
johnc@1325 | 113 | size_t max_card_num = ((size_t)1 << (sizeof(unsigned)*BitsPerByte-1)) - 1; |
johnc@2713 | 114 | guarantee(_max_cards < max_card_num, "card_num representation"); |
johnc@1325 | 115 | |
johnc@2713 | 116 | // We need _n_card_counts to be less than _max_n_card_counts here |
johnc@2713 | 117 | // so that the expansion call (below) actually allocates the |
johnc@2713 | 118 | // _counts and _epochs arrays. |
johnc@2713 | 119 | assert(_n_card_counts == 0, "pre-condition"); |
johnc@2713 | 120 | assert(_max_n_card_counts > 0, "pre-condition"); |
johnc@2713 | 121 | |
johnc@2713 | 122 | // Find the index into cache size array that is of a size that's |
johnc@2713 | 123 | // large enough to hold desired_sz. |
johnc@2713 | 124 | size_t desired_sz = _max_cards / InitialCacheFraction; |
johnc@2713 | 125 | int desired_sz_index = 0; |
johnc@2713 | 126 | while (_cc_cache_sizes[desired_sz_index] < desired_sz) { |
johnc@2713 | 127 | desired_sz_index += 1; |
johnc@2713 | 128 | assert(desired_sz_index < MAX_CC_CACHE_INDEX, "invariant"); |
johnc@1325 | 129 | } |
johnc@2713 | 130 | assert(desired_sz_index < MAX_CC_CACHE_INDEX, "invariant"); |
johnc@1325 | 131 | |
johnc@2713 | 132 | // If the desired_sz value is between two sizes then |
johnc@2713 | 133 | // _cc_cache_sizes[desired_sz_index-1] < desired_sz <= _cc_cache_sizes[desired_sz_index] |
johnc@2713 | 134 | // we will start with the lower size in the optimistic expectation that |
johnc@2713 | 135 | // we will not need to expand up. Note desired_sz_index could also be 0. |
johnc@2713 | 136 | if (desired_sz_index > 0 && |
johnc@2713 | 137 | _cc_cache_sizes[desired_sz_index] > desired_sz) { |
johnc@2713 | 138 | desired_sz_index -= 1; |
johnc@2713 | 139 | } |
johnc@1325 | 140 | |
johnc@2713 | 141 | if (!expand_card_count_cache(desired_sz_index)) { |
johnc@2713 | 142 | // Allocation was unsuccessful - exit |
johnc@2713 | 143 | vm_exit_during_initialization("Could not reserve enough space for card count cache"); |
johnc@2713 | 144 | } |
johnc@2713 | 145 | assert(_n_card_counts > 0, "post-condition"); |
johnc@2713 | 146 | assert(_cache_size_index == desired_sz_index, "post-condition"); |
johnc@1325 | 147 | |
johnc@1325 | 148 | Copy::fill_to_bytes(&_card_counts[0], |
johnc@1325 | 149 | _n_card_counts * sizeof(CardCountCacheEntry)); |
johnc@1325 | 150 | Copy::fill_to_bytes(&_card_epochs[0], _n_card_counts * sizeof(CardEpochCacheEntry)); |
johnc@1325 | 151 | |
johnc@1325 | 152 | ModRefBarrierSet* bs = _g1h->mr_bs(); |
ysr@777 | 153 | guarantee(bs->is_a(BarrierSet::CardTableModRef), "Precondition"); |
johnc@1325 | 154 | _ct_bs = (CardTableModRefBS*)bs; |
johnc@1325 | 155 | _ct_bot = _ct_bs->byte_for_const(_g1h->reserved_region().start()); |
johnc@1325 | 156 | |
ysr@777 | 157 | _def_use_cache = true; |
ysr@777 | 158 | _use_cache = true; |
ysr@777 | 159 | _hot_cache_size = (1 << G1ConcRSLogCacheSize); |
ysr@777 | 160 | _hot_cache = NEW_C_HEAP_ARRAY(jbyte*, _hot_cache_size); |
ysr@777 | 161 | _n_hot = 0; |
ysr@777 | 162 | _hot_cache_idx = 0; |
johnc@1324 | 163 | |
johnc@1324 | 164 | // For refining the cards in the hot cache in parallel |
johnc@1324 | 165 | int n_workers = (ParallelGCThreads > 0 ? |
johnc@1325 | 166 | _g1h->workers()->total_workers() : 1); |
johnc@1324 | 167 | _hot_cache_par_chunk_size = MAX2(1, _hot_cache_size / n_workers); |
johnc@1324 | 168 | _hot_cache_par_claimed_idx = 0; |
ysr@777 | 169 | } |
ysr@777 | 170 | } |
ysr@777 | 171 | |
iveresov@1229 | 172 | void ConcurrentG1Refine::stop() { |
iveresov@1229 | 173 | if (_threads != NULL) { |
iveresov@1229 | 174 | for (int i = 0; i < _n_threads; i++) { |
iveresov@1229 | 175 | _threads[i]->stop(); |
iveresov@1229 | 176 | } |
iveresov@1229 | 177 | } |
iveresov@1229 | 178 | } |
iveresov@1229 | 179 | |
iveresov@1546 | 180 | void ConcurrentG1Refine::reinitialize_threads() { |
iveresov@1546 | 181 | reset_threshold_step(); |
iveresov@1546 | 182 | if (_threads != NULL) { |
iveresov@1546 | 183 | for (int i = 0; i < _n_threads; i++) { |
iveresov@1546 | 184 | _threads[i]->initialize(); |
iveresov@1546 | 185 | } |
iveresov@1546 | 186 | } |
iveresov@1546 | 187 | } |
iveresov@1546 | 188 | |
ysr@777 | 189 | ConcurrentG1Refine::~ConcurrentG1Refine() { |
johnc@1325 | 190 | if (G1ConcRSLogCacheSize > 0) { |
johnc@2713 | 191 | // Please see the comment in allocate_card_count_cache |
johnc@2713 | 192 | // for why we call os::malloc() and os::free() directly. |
ysr@777 | 193 | assert(_card_counts != NULL, "Logic"); |
johnc@2713 | 194 | os::free(_card_counts); |
johnc@1325 | 195 | assert(_card_epochs != NULL, "Logic"); |
johnc@2713 | 196 | os::free(_card_epochs); |
johnc@2713 | 197 | |
ysr@777 | 198 | assert(_hot_cache != NULL, "Logic"); |
ysr@777 | 199 | FREE_C_HEAP_ARRAY(jbyte*, _hot_cache); |
ysr@777 | 200 | } |
iveresov@1229 | 201 | if (_threads != NULL) { |
iveresov@1229 | 202 | for (int i = 0; i < _n_threads; i++) { |
iveresov@1229 | 203 | delete _threads[i]; |
iveresov@1229 | 204 | } |
iveresov@1234 | 205 | FREE_C_HEAP_ARRAY(ConcurrentG1RefineThread*, _threads); |
ysr@777 | 206 | } |
ysr@777 | 207 | } |
ysr@777 | 208 | |
iveresov@1229 | 209 | void ConcurrentG1Refine::threads_do(ThreadClosure *tc) { |
iveresov@1229 | 210 | if (_threads != NULL) { |
iveresov@1229 | 211 | for (int i = 0; i < _n_threads; i++) { |
iveresov@1229 | 212 | tc->do_thread(_threads[i]); |
iveresov@1229 | 213 | } |
ysr@777 | 214 | } |
ysr@777 | 215 | } |
ysr@777 | 216 | |
johnc@1325 | 217 | bool ConcurrentG1Refine::is_young_card(jbyte* card_ptr) { |
johnc@1325 | 218 | HeapWord* start = _ct_bs->addr_for(card_ptr); |
johnc@1325 | 219 | HeapRegion* r = _g1h->heap_region_containing(start); |
johnc@1325 | 220 | if (r != NULL && r->is_young()) { |
johnc@1325 | 221 | return true; |
johnc@1325 | 222 | } |
johnc@1325 | 223 | // This card is not associated with a heap region |
johnc@1325 | 224 | // so can't be young. |
johnc@1325 | 225 | return false; |
ysr@777 | 226 | } |
ysr@777 | 227 | |
johnc@1325 | 228 | jbyte* ConcurrentG1Refine::add_card_count(jbyte* card_ptr, int* count, bool* defer) { |
johnc@1325 | 229 | unsigned new_card_num = ptr_2_card_num(card_ptr); |
johnc@1325 | 230 | unsigned bucket = hash(new_card_num); |
johnc@1325 | 231 | assert(0 <= bucket && bucket < _n_card_counts, "Bounds"); |
johnc@1325 | 232 | |
johnc@1325 | 233 | CardCountCacheEntry* count_ptr = &_card_counts[bucket]; |
johnc@1325 | 234 | CardEpochCacheEntry* epoch_ptr = &_card_epochs[bucket]; |
johnc@1325 | 235 | |
johnc@1325 | 236 | // We have to construct a new entry if we haven't updated the counts |
johnc@1325 | 237 | // during the current period, or if the count was updated for a |
johnc@1325 | 238 | // different card number. |
johnc@1325 | 239 | unsigned int new_epoch = (unsigned int) _n_periods; |
johnc@1325 | 240 | julong new_epoch_entry = make_epoch_entry(new_card_num, new_epoch); |
johnc@1325 | 241 | |
johnc@1325 | 242 | while (true) { |
johnc@1325 | 243 | // Fetch the previous epoch value |
johnc@1325 | 244 | julong prev_epoch_entry = epoch_ptr->_value; |
johnc@1325 | 245 | julong cas_res; |
johnc@1325 | 246 | |
johnc@1325 | 247 | if (extract_epoch(prev_epoch_entry) != new_epoch) { |
johnc@1325 | 248 | // This entry has not yet been updated during this period. |
johnc@1325 | 249 | // Note: we update the epoch value atomically to ensure |
johnc@1325 | 250 | // that there is only one winner that updates the cached |
johnc@1325 | 251 | // card_ptr value even though all the refine threads share |
johnc@1325 | 252 | // the same epoch value. |
johnc@1325 | 253 | |
johnc@1325 | 254 | cas_res = (julong) Atomic::cmpxchg((jlong) new_epoch_entry, |
johnc@1325 | 255 | (volatile jlong*)&epoch_ptr->_value, |
johnc@1325 | 256 | (jlong) prev_epoch_entry); |
johnc@1325 | 257 | |
johnc@1325 | 258 | if (cas_res == prev_epoch_entry) { |
johnc@1325 | 259 | // We have successfully won the race to update the |
johnc@1325 | 260 | // epoch and card_num value. Make it look like the |
johnc@1325 | 261 | // count and eviction count were previously cleared. |
johnc@1325 | 262 | count_ptr->_count = 1; |
johnc@1325 | 263 | count_ptr->_evict_count = 0; |
johnc@1325 | 264 | *count = 0; |
johnc@1325 | 265 | // We can defer the processing of card_ptr |
johnc@1325 | 266 | *defer = true; |
johnc@1325 | 267 | return card_ptr; |
johnc@1325 | 268 | } |
johnc@1325 | 269 | // We did not win the race to update the epoch field, so some other |
johnc@1325 | 270 | // thread must have done it. The value that gets returned by CAS |
johnc@1325 | 271 | // should be the new epoch value. |
johnc@1325 | 272 | assert(extract_epoch(cas_res) == new_epoch, "unexpected epoch"); |
johnc@1325 | 273 | // We could 'continue' here or just re-read the previous epoch value |
johnc@1325 | 274 | prev_epoch_entry = epoch_ptr->_value; |
johnc@1325 | 275 | } |
johnc@1325 | 276 | |
johnc@1325 | 277 | // The epoch entry for card_ptr has been updated during this period. |
johnc@1325 | 278 | unsigned old_card_num = extract_card_num(prev_epoch_entry); |
johnc@1325 | 279 | |
johnc@1325 | 280 | // The card count that will be returned to caller |
johnc@1325 | 281 | *count = count_ptr->_count; |
johnc@1325 | 282 | |
johnc@1325 | 283 | // Are we updating the count for the same card? |
johnc@1325 | 284 | if (new_card_num == old_card_num) { |
johnc@1325 | 285 | // Same card - just update the count. We could have more than one |
johnc@1325 | 286 | // thread racing to update count for the current card. It should be |
johnc@1325 | 287 | // OK not to use a CAS as the only penalty should be some missed |
johnc@1325 | 288 | // increments of the count which delays identifying the card as "hot". |
johnc@1325 | 289 | |
johnc@1325 | 290 | if (*count < max_jubyte) count_ptr->_count++; |
johnc@1325 | 291 | // We can defer the processing of card_ptr |
johnc@1325 | 292 | *defer = true; |
johnc@1325 | 293 | return card_ptr; |
johnc@1325 | 294 | } |
johnc@1325 | 295 | |
johnc@1325 | 296 | // Different card - evict old card info |
johnc@1325 | 297 | if (count_ptr->_evict_count < max_jubyte) count_ptr->_evict_count++; |
johnc@1325 | 298 | if (count_ptr->_evict_count > G1CardCountCacheExpandThreshold) { |
johnc@1325 | 299 | // Trigger a resize the next time we clear |
johnc@1325 | 300 | _expand_card_counts = true; |
johnc@1325 | 301 | } |
johnc@1325 | 302 | |
johnc@1325 | 303 | cas_res = (julong) Atomic::cmpxchg((jlong) new_epoch_entry, |
johnc@1325 | 304 | (volatile jlong*)&epoch_ptr->_value, |
johnc@1325 | 305 | (jlong) prev_epoch_entry); |
johnc@1325 | 306 | |
johnc@1325 | 307 | if (cas_res == prev_epoch_entry) { |
johnc@1325 | 308 | // We successfully updated the card num value in the epoch entry |
johnc@1325 | 309 | count_ptr->_count = 0; // initialize counter for new card num |
johnc@2021 | 310 | jbyte* old_card_ptr = card_num_2_ptr(old_card_num); |
johnc@1325 | 311 | |
johnc@1325 | 312 | // Even though the region containg the card at old_card_num was not |
johnc@1325 | 313 | // in the young list when old_card_num was recorded in the epoch |
johnc@1325 | 314 | // cache it could have been added to the free list and subsequently |
johnc@2021 | 315 | // added to the young list in the intervening time. See CR 6817995. |
johnc@2021 | 316 | // We do not deal with this case here - it will be handled in |
johnc@2021 | 317 | // HeapRegion::oops_on_card_seq_iterate_careful after it has been |
johnc@2021 | 318 | // determined that the region containing the card has been allocated |
johnc@2021 | 319 | // to, and it's safe to check the young type of the region. |
johnc@1325 | 320 | |
johnc@1325 | 321 | // We do not want to defer processing of card_ptr in this case |
johnc@1325 | 322 | // (we need to refine old_card_ptr and card_ptr) |
johnc@1325 | 323 | *defer = false; |
johnc@1325 | 324 | return old_card_ptr; |
johnc@1325 | 325 | } |
johnc@1325 | 326 | // Someone else beat us - try again. |
johnc@1325 | 327 | } |
johnc@1325 | 328 | } |
johnc@1325 | 329 | |
johnc@1325 | 330 | jbyte* ConcurrentG1Refine::cache_insert(jbyte* card_ptr, bool* defer) { |
johnc@1325 | 331 | int count; |
johnc@1325 | 332 | jbyte* cached_ptr = add_card_count(card_ptr, &count, defer); |
johnc@1325 | 333 | assert(cached_ptr != NULL, "bad cached card ptr"); |
johnc@1681 | 334 | |
johnc@2021 | 335 | // We've just inserted a card pointer into the card count cache |
johnc@2021 | 336 | // and got back the card that we just inserted or (evicted) the |
johnc@2021 | 337 | // previous contents of that count slot. |
johnc@1681 | 338 | |
johnc@2021 | 339 | // The card we got back could be in a young region. When the |
johnc@2021 | 340 | // returned card (if evicted) was originally inserted, we had |
johnc@2021 | 341 | // determined that its containing region was not young. However |
johnc@2021 | 342 | // it is possible for the region to be freed during a cleanup |
johnc@2021 | 343 | // pause, then reallocated and tagged as young which will result |
johnc@2021 | 344 | // in the returned card residing in a young region. |
johnc@2021 | 345 | // |
johnc@2021 | 346 | // We do not deal with this case here - the change from non-young |
johnc@2021 | 347 | // to young could be observed at any time - it will be handled in |
johnc@2021 | 348 | // HeapRegion::oops_on_card_seq_iterate_careful after it has been |
johnc@2021 | 349 | // determined that the region containing the card has been allocated |
johnc@2021 | 350 | // to. |
johnc@1325 | 351 | |
johnc@1325 | 352 | // The card pointer we obtained from card count cache is not hot |
johnc@1325 | 353 | // so do not store it in the cache; return it for immediate |
johnc@1325 | 354 | // refining. |
ysr@777 | 355 | if (count < G1ConcRSHotCardLimit) { |
johnc@1325 | 356 | return cached_ptr; |
ysr@777 | 357 | } |
johnc@1325 | 358 | |
johnc@2021 | 359 | // Otherwise, the pointer we got from the _card_counts cache is hot. |
ysr@777 | 360 | jbyte* res = NULL; |
ysr@777 | 361 | MutexLockerEx x(HotCardCache_lock, Mutex::_no_safepoint_check_flag); |
ysr@777 | 362 | if (_n_hot == _hot_cache_size) { |
ysr@777 | 363 | res = _hot_cache[_hot_cache_idx]; |
ysr@777 | 364 | _n_hot--; |
ysr@777 | 365 | } |
ysr@777 | 366 | // Now _n_hot < _hot_cache_size, and we can insert at _hot_cache_idx. |
johnc@1325 | 367 | _hot_cache[_hot_cache_idx] = cached_ptr; |
ysr@777 | 368 | _hot_cache_idx++; |
ysr@777 | 369 | if (_hot_cache_idx == _hot_cache_size) _hot_cache_idx = 0; |
ysr@777 | 370 | _n_hot++; |
johnc@1325 | 371 | |
johnc@2021 | 372 | // The card obtained from the hot card cache could be in a young |
johnc@2021 | 373 | // region. See above on how this can happen. |
johnc@1325 | 374 | |
ysr@777 | 375 | return res; |
ysr@777 | 376 | } |
ysr@777 | 377 | |
johnc@2060 | 378 | void ConcurrentG1Refine::clean_up_cache(int worker_i, |
johnc@2060 | 379 | G1RemSet* g1rs, |
johnc@2060 | 380 | DirtyCardQueue* into_cset_dcq) { |
ysr@777 | 381 | assert(!use_cache(), "cache should be disabled"); |
johnc@1324 | 382 | int start_idx; |
johnc@1324 | 383 | |
johnc@1324 | 384 | while ((start_idx = _hot_cache_par_claimed_idx) < _n_hot) { // read once |
johnc@1324 | 385 | int end_idx = start_idx + _hot_cache_par_chunk_size; |
johnc@1324 | 386 | |
johnc@1324 | 387 | if (start_idx == |
johnc@1324 | 388 | Atomic::cmpxchg(end_idx, &_hot_cache_par_claimed_idx, start_idx)) { |
johnc@1324 | 389 | // The current worker has successfully claimed the chunk [start_idx..end_idx) |
johnc@1324 | 390 | end_idx = MIN2(end_idx, _n_hot); |
johnc@1324 | 391 | for (int i = start_idx; i < end_idx; i++) { |
johnc@1324 | 392 | jbyte* entry = _hot_cache[i]; |
johnc@1324 | 393 | if (entry != NULL) { |
johnc@2060 | 394 | if (g1rs->concurrentRefineOneCard(entry, worker_i, true)) { |
johnc@2060 | 395 | // 'entry' contains references that point into the current |
johnc@2060 | 396 | // collection set. We need to record 'entry' in the DCQS |
johnc@2060 | 397 | // that's used for that purpose. |
johnc@2060 | 398 | // |
johnc@2060 | 399 | // The only time we care about recording cards that contain |
johnc@2060 | 400 | // references that point into the collection set is during |
johnc@2060 | 401 | // RSet updating while within an evacuation pause. |
johnc@2060 | 402 | // In this case worker_i should be the id of a GC worker thread |
johnc@2060 | 403 | assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause"); |
brutisso@2646 | 404 | assert(worker_i < (int) (ParallelGCThreads == 0 ? 1 : ParallelGCThreads), "incorrect worker id"); |
johnc@2060 | 405 | into_cset_dcq->enqueue(entry); |
johnc@2060 | 406 | } |
johnc@1324 | 407 | } |
johnc@1324 | 408 | } |
ysr@777 | 409 | } |
ysr@777 | 410 | } |
ysr@777 | 411 | } |
ysr@777 | 412 | |
johnc@2713 | 413 | // The arrays used to hold the card counts and the epochs must have |
johnc@2713 | 414 | // a 1:1 correspondence. Hence they are allocated and freed together |
johnc@2713 | 415 | // Returns true if the allocations of both the counts and epochs |
johnc@2713 | 416 | // were successful; false otherwise. |
johnc@2713 | 417 | bool ConcurrentG1Refine::allocate_card_count_cache(size_t n, |
johnc@2713 | 418 | CardCountCacheEntry** counts, |
johnc@2713 | 419 | CardEpochCacheEntry** epochs) { |
johnc@2713 | 420 | // We call the allocation/free routines directly for the counts |
johnc@2713 | 421 | // and epochs arrays. The NEW_C_HEAP_ARRAY/FREE_C_HEAP_ARRAY |
johnc@2713 | 422 | // macros call AllocateHeap and FreeHeap respectively. |
johnc@2713 | 423 | // AllocateHeap will call vm_exit_out_of_memory in the event |
johnc@2713 | 424 | // of an allocation failure and abort the JVM. With the |
johnc@2713 | 425 | // _counts/epochs arrays we only need to abort the JVM if the |
johnc@2713 | 426 | // initial allocation of these arrays fails. |
johnc@2713 | 427 | // |
johnc@2713 | 428 | // Additionally AllocateHeap/FreeHeap do some tracing of |
johnc@2713 | 429 | // allocate/free calls so calling one without calling the |
johnc@2713 | 430 | // other can cause inconsistencies in the tracing. So we |
johnc@2713 | 431 | // call neither. |
johnc@2713 | 432 | |
johnc@2713 | 433 | assert(*counts == NULL, "out param"); |
johnc@2713 | 434 | assert(*epochs == NULL, "out param"); |
johnc@2713 | 435 | |
johnc@2713 | 436 | size_t counts_size = n * sizeof(CardCountCacheEntry); |
johnc@2713 | 437 | size_t epochs_size = n * sizeof(CardEpochCacheEntry); |
johnc@2713 | 438 | |
johnc@2713 | 439 | *counts = (CardCountCacheEntry*) os::malloc(counts_size); |
johnc@2713 | 440 | if (*counts == NULL) { |
johnc@2713 | 441 | // allocation was unsuccessful |
johnc@2713 | 442 | return false; |
johnc@2713 | 443 | } |
johnc@2713 | 444 | |
johnc@2713 | 445 | *epochs = (CardEpochCacheEntry*) os::malloc(epochs_size); |
johnc@2713 | 446 | if (*epochs == NULL) { |
johnc@2713 | 447 | // allocation was unsuccessful - free counts array |
johnc@2713 | 448 | assert(*counts != NULL, "must be"); |
johnc@2713 | 449 | os::free(*counts); |
johnc@2713 | 450 | *counts = NULL; |
johnc@2713 | 451 | return false; |
johnc@2713 | 452 | } |
johnc@2713 | 453 | |
johnc@2713 | 454 | // We successfully allocated both counts and epochs |
johnc@2713 | 455 | return true; |
johnc@2713 | 456 | } |
johnc@2713 | 457 | |
johnc@2713 | 458 | // Returns true if the card counts/epochs cache was |
johnc@2713 | 459 | // successfully expanded; false otherwise. |
johnc@2713 | 460 | bool ConcurrentG1Refine::expand_card_count_cache(int cache_size_idx) { |
johnc@2713 | 461 | // Can we expand the card count and epoch tables? |
johnc@1325 | 462 | if (_n_card_counts < _max_n_card_counts) { |
johnc@2713 | 463 | assert(cache_size_idx >= 0 && cache_size_idx < MAX_CC_CACHE_INDEX, "oob"); |
johnc@1325 | 464 | |
johnc@2713 | 465 | size_t cache_size = _cc_cache_sizes[cache_size_idx]; |
johnc@1325 | 466 | // Make sure we don't go bigger than we will ever need |
johnc@2713 | 467 | cache_size = MIN2(cache_size, _max_n_card_counts); |
johnc@1325 | 468 | |
johnc@2713 | 469 | // Should we expand the card count and card epoch tables? |
johnc@2713 | 470 | if (cache_size > _n_card_counts) { |
johnc@2713 | 471 | // We have been asked to allocate new, larger, arrays for |
johnc@2713 | 472 | // the card counts and the epochs. Attempt the allocation |
johnc@2713 | 473 | // of both before we free the existing arrays in case |
johnc@2713 | 474 | // the allocation is unsuccessful... |
johnc@2713 | 475 | CardCountCacheEntry* counts = NULL; |
johnc@2713 | 476 | CardEpochCacheEntry* epochs = NULL; |
johnc@2713 | 477 | |
johnc@2713 | 478 | if (allocate_card_count_cache(cache_size, &counts, &epochs)) { |
johnc@2713 | 479 | // Allocation was successful. |
johnc@2713 | 480 | // We can just free the old arrays; we're |
johnc@2713 | 481 | // not interested in preserving the contents |
johnc@2713 | 482 | if (_card_counts != NULL) os::free(_card_counts); |
johnc@2713 | 483 | if (_card_epochs != NULL) os::free(_card_epochs); |
johnc@2713 | 484 | |
johnc@2713 | 485 | // Cache the size of the arrays and the index that got us there. |
johnc@2713 | 486 | _n_card_counts = cache_size; |
johnc@2713 | 487 | _cache_size_index = cache_size_idx; |
johnc@2713 | 488 | |
johnc@2713 | 489 | _card_counts = counts; |
johnc@2713 | 490 | _card_epochs = epochs; |
johnc@2713 | 491 | |
johnc@2713 | 492 | // We successfully allocated/expanded the caches. |
johnc@2713 | 493 | return true; |
johnc@2713 | 494 | } |
ysr@777 | 495 | } |
ysr@777 | 496 | } |
johnc@2713 | 497 | |
johnc@2713 | 498 | // We did not successfully expand the caches. |
johnc@2713 | 499 | return false; |
ysr@777 | 500 | } |
ysr@777 | 501 | |
johnc@1325 | 502 | void ConcurrentG1Refine::clear_and_record_card_counts() { |
johnc@1325 | 503 | if (G1ConcRSLogCacheSize == 0) return; |
johnc@1325 | 504 | |
johnc@1325 | 505 | #ifndef PRODUCT |
johnc@1325 | 506 | double start = os::elapsedTime(); |
johnc@1325 | 507 | #endif |
johnc@1325 | 508 | |
johnc@1325 | 509 | if (_expand_card_counts) { |
johnc@2713 | 510 | int new_idx = _cache_size_index + 1; |
johnc@2713 | 511 | |
johnc@2713 | 512 | if (expand_card_count_cache(new_idx)) { |
johnc@2713 | 513 | // Allocation was successful and _n_card_counts has |
johnc@2713 | 514 | // been updated to the new size. We only need to clear |
johnc@2713 | 515 | // the epochs so we don't read a bogus epoch value |
johnc@2713 | 516 | // when inserting a card into the hot card cache. |
johnc@2713 | 517 | Copy::fill_to_bytes(&_card_epochs[0], _n_card_counts * sizeof(CardEpochCacheEntry)); |
johnc@2713 | 518 | } |
johnc@1325 | 519 | _expand_card_counts = false; |
ysr@777 | 520 | } |
ysr@777 | 521 | |
johnc@1325 | 522 | int this_epoch = (int) _n_periods; |
johnc@1325 | 523 | assert((this_epoch+1) <= max_jint, "to many periods"); |
johnc@1325 | 524 | // Update epoch |
johnc@1325 | 525 | _n_periods++; |
johnc@1325 | 526 | |
johnc@1325 | 527 | #ifndef PRODUCT |
johnc@1325 | 528 | double elapsed = os::elapsedTime() - start; |
johnc@1325 | 529 | _g1h->g1_policy()->record_cc_clear_time(elapsed * 1000.0); |
johnc@1325 | 530 | #endif |
ysr@777 | 531 | } |
tonyp@1454 | 532 | |
tonyp@1454 | 533 | void ConcurrentG1Refine::print_worker_threads_on(outputStream* st) const { |
tonyp@1454 | 534 | for (int i = 0; i < _n_threads; ++i) { |
tonyp@1454 | 535 | _threads[i]->print_on(st); |
tonyp@1454 | 536 | st->cr(); |
tonyp@1454 | 537 | } |
tonyp@1454 | 538 | } |