1.1 --- a/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp Wed Mar 23 14:12:51 2011 +0100
1.2 +++ b/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp Mon Mar 28 10:58:54 2011 -0700
1.3 @@ -31,19 +31,23 @@
1.4 #include "gc_implementation/g1/heapRegionSeq.inline.hpp"
1.5 #include "memory/space.inline.hpp"
1.6 #include "runtime/atomic.hpp"
1.7 +#include "runtime/java.hpp"
1.8 #include "utilities/copy.hpp"
1.9
1.10 // Possible sizes for the card counts cache: odd primes that roughly double in size.
1.11 // (See jvmtiTagMap.cpp).
1.12 -int ConcurrentG1Refine::_cc_cache_sizes[] = {
1.13 - 16381, 32771, 76831, 150001, 307261,
1.14 - 614563, 1228891, 2457733, 4915219, 9830479,
1.15 - 19660831, 39321619, 78643219, 157286461, -1
1.16 +
1.17 +#define MAX_SIZE ((size_t) -1)
1.18 +
1.19 +size_t ConcurrentG1Refine::_cc_cache_sizes[] = {
1.20 + 16381, 32771, 76831, 150001, 307261,
1.21 + 614563, 1228891, 2457733, 4915219, 9830479,
1.22 + 19660831, 39321619, 78643219, 157286461, MAX_SIZE
1.23 };
1.24
1.25 ConcurrentG1Refine::ConcurrentG1Refine() :
1.26 _card_counts(NULL), _card_epochs(NULL),
1.27 - _n_card_counts(0), _max_n_card_counts(0),
1.28 + _n_card_counts(0), _max_cards(0), _max_n_card_counts(0),
1.29 _cache_size_index(0), _expand_card_counts(false),
1.30 _hot_cache(NULL),
1.31 _def_use_cache(false), _use_cache(false),
1.32 @@ -98,27 +102,44 @@
1.33 void ConcurrentG1Refine::init() {
1.34 if (G1ConcRSLogCacheSize > 0) {
1.35 _g1h = G1CollectedHeap::heap();
1.36 - _max_n_card_counts =
1.37 - (unsigned) (_g1h->max_capacity() >> CardTableModRefBS::card_shift);
1.38 +
1.39 + _max_cards = _g1h->max_capacity() >> CardTableModRefBS::card_shift;
1.40 + _max_n_card_counts = _max_cards * G1MaxHotCardCountSizePercent / 100;
1.41
1.42 size_t max_card_num = ((size_t)1 << (sizeof(unsigned)*BitsPerByte-1)) - 1;
1.43 - guarantee(_max_n_card_counts < max_card_num, "card_num representation");
1.44 + guarantee(_max_cards < max_card_num, "card_num representation");
1.45
1.46 - int desired = _max_n_card_counts / InitialCacheFraction;
1.47 - for (_cache_size_index = 0;
1.48 - _cc_cache_sizes[_cache_size_index] >= 0; _cache_size_index++) {
1.49 - if (_cc_cache_sizes[_cache_size_index] >= desired) break;
1.50 + // We need _n_card_counts to be less than _max_n_card_counts here
1.51 + // so that the expansion call (below) actually allocates the
1.52 + // _counts and _epochs arrays.
1.53 + assert(_n_card_counts == 0, "pre-condition");
1.54 + assert(_max_n_card_counts > 0, "pre-condition");
1.55 +
1.56 + // Find the index into cache size array that is of a size that's
1.57 + // large enough to hold desired_sz.
1.58 + size_t desired_sz = _max_cards / InitialCacheFraction;
1.59 + int desired_sz_index = 0;
1.60 + while (_cc_cache_sizes[desired_sz_index] < desired_sz) {
1.61 + desired_sz_index += 1;
1.62 + assert(desired_sz_index < MAX_CC_CACHE_INDEX, "invariant");
1.63 }
1.64 - _cache_size_index = MAX2(0, (_cache_size_index - 1));
1.65 + assert(desired_sz_index < MAX_CC_CACHE_INDEX, "invariant");
1.66
1.67 - int initial_size = _cc_cache_sizes[_cache_size_index];
1.68 - if (initial_size < 0) initial_size = _max_n_card_counts;
1.69 + // If the desired_sz value is between two sizes then
1.70 + // _cc_cache_sizes[desired_sz_index-1] < desired_sz <= _cc_cache_sizes[desired_sz_index]
1.71 + // we will start with the lower size in the optimistic expectation that
1.72 + // we will not need to expand up. Note desired_sz_index could also be 0.
1.73 + if (desired_sz_index > 0 &&
1.74 + _cc_cache_sizes[desired_sz_index] > desired_sz) {
1.75 + desired_sz_index -= 1;
1.76 + }
1.77
1.78 - // Make sure we don't go bigger than we will ever need
1.79 - _n_card_counts = MIN2((unsigned) initial_size, _max_n_card_counts);
1.80 -
1.81 - _card_counts = NEW_C_HEAP_ARRAY(CardCountCacheEntry, _n_card_counts);
1.82 - _card_epochs = NEW_C_HEAP_ARRAY(CardEpochCacheEntry, _n_card_counts);
1.83 + if (!expand_card_count_cache(desired_sz_index)) {
1.84 + // Allocation was unsuccessful - exit
1.85 + vm_exit_during_initialization("Could not reserve enough space for card count cache");
1.86 + }
1.87 + assert(_n_card_counts > 0, "post-condition");
1.88 + assert(_cache_size_index == desired_sz_index, "post-condition");
1.89
1.90 Copy::fill_to_bytes(&_card_counts[0],
1.91 _n_card_counts * sizeof(CardCountCacheEntry));
1.92 @@ -163,10 +184,13 @@
1.93
1.94 ConcurrentG1Refine::~ConcurrentG1Refine() {
1.95 if (G1ConcRSLogCacheSize > 0) {
1.96 + // Please see the comment in allocate_card_count_cache
1.97 + // for why we call os::malloc() and os::free() directly.
1.98 assert(_card_counts != NULL, "Logic");
1.99 - FREE_C_HEAP_ARRAY(CardCountCacheEntry, _card_counts);
1.100 + os::free(_card_counts);
1.101 assert(_card_epochs != NULL, "Logic");
1.102 - FREE_C_HEAP_ARRAY(CardEpochCacheEntry, _card_epochs);
1.103 + os::free(_card_epochs);
1.104 +
1.105 assert(_hot_cache != NULL, "Logic");
1.106 FREE_C_HEAP_ARRAY(jbyte*, _hot_cache);
1.107 }
1.108 @@ -382,29 +406,93 @@
1.109 }
1.110 }
1.111
1.112 -void ConcurrentG1Refine::expand_card_count_cache() {
1.113 +// The arrays used to hold the card counts and the epochs must have
1.114 +// a 1:1 correspondence. Hence they are allocated and freed together
1.115 +// Returns true if the allocations of both the counts and epochs
1.116 +// were successful; false otherwise.
1.117 +bool ConcurrentG1Refine::allocate_card_count_cache(size_t n,
1.118 + CardCountCacheEntry** counts,
1.119 + CardEpochCacheEntry** epochs) {
1.120 + // We call the allocation/free routines directly for the counts
1.121 + // and epochs arrays. The NEW_C_HEAP_ARRAY/FREE_C_HEAP_ARRAY
1.122 + // macros call AllocateHeap and FreeHeap respectively.
1.123 + // AllocateHeap will call vm_exit_out_of_memory in the event
1.124 + // of an allocation failure and abort the JVM. With the
1.125 + // _counts/epochs arrays we only need to abort the JVM if the
1.126 + // initial allocation of these arrays fails.
1.127 + //
1.128 + // Additionally AllocateHeap/FreeHeap do some tracing of
1.129 + // allocate/free calls so calling one without calling the
1.130 + // other can cause inconsistencies in the tracing. So we
1.131 + // call neither.
1.132 +
1.133 + assert(*counts == NULL, "out param");
1.134 + assert(*epochs == NULL, "out param");
1.135 +
1.136 + size_t counts_size = n * sizeof(CardCountCacheEntry);
1.137 + size_t epochs_size = n * sizeof(CardEpochCacheEntry);
1.138 +
1.139 + *counts = (CardCountCacheEntry*) os::malloc(counts_size);
1.140 + if (*counts == NULL) {
1.141 + // allocation was unsuccessful
1.142 + return false;
1.143 + }
1.144 +
1.145 + *epochs = (CardEpochCacheEntry*) os::malloc(epochs_size);
1.146 + if (*epochs == NULL) {
1.147 + // allocation was unsuccessful - free counts array
1.148 + assert(*counts != NULL, "must be");
1.149 + os::free(*counts);
1.150 + *counts = NULL;
1.151 + return false;
1.152 + }
1.153 +
1.154 + // We successfully allocated both counts and epochs
1.155 + return true;
1.156 +}
1.157 +
1.158 +// Returns true if the card counts/epochs cache was
1.159 +// successfully expanded; false otherwise.
1.160 +bool ConcurrentG1Refine::expand_card_count_cache(int cache_size_idx) {
1.161 + // Can we expand the card count and epoch tables?
1.162 if (_n_card_counts < _max_n_card_counts) {
1.163 - int new_idx = _cache_size_index+1;
1.164 - int new_size = _cc_cache_sizes[new_idx];
1.165 - if (new_size < 0) new_size = _max_n_card_counts;
1.166 + assert(cache_size_idx >= 0 && cache_size_idx < MAX_CC_CACHE_INDEX, "oob");
1.167
1.168 + size_t cache_size = _cc_cache_sizes[cache_size_idx];
1.169 // Make sure we don't go bigger than we will ever need
1.170 - new_size = MIN2((unsigned) new_size, _max_n_card_counts);
1.171 + cache_size = MIN2(cache_size, _max_n_card_counts);
1.172
1.173 - // Expand the card count and card epoch tables
1.174 - if (new_size > (int)_n_card_counts) {
1.175 - // We can just free and allocate a new array as we're
1.176 - // not interested in preserving the contents
1.177 - assert(_card_counts != NULL, "Logic!");
1.178 - assert(_card_epochs != NULL, "Logic!");
1.179 - FREE_C_HEAP_ARRAY(CardCountCacheEntry, _card_counts);
1.180 - FREE_C_HEAP_ARRAY(CardEpochCacheEntry, _card_epochs);
1.181 - _n_card_counts = new_size;
1.182 - _card_counts = NEW_C_HEAP_ARRAY(CardCountCacheEntry, _n_card_counts);
1.183 - _card_epochs = NEW_C_HEAP_ARRAY(CardEpochCacheEntry, _n_card_counts);
1.184 - _cache_size_index = new_idx;
1.185 + // Should we expand the card count and card epoch tables?
1.186 + if (cache_size > _n_card_counts) {
1.187 + // We have been asked to allocate new, larger, arrays for
1.188 + // the card counts and the epochs. Attempt the allocation
1.189 + // of both before we free the existing arrays in case
1.190 + // the allocation is unsuccessful...
1.191 + CardCountCacheEntry* counts = NULL;
1.192 + CardEpochCacheEntry* epochs = NULL;
1.193 +
1.194 + if (allocate_card_count_cache(cache_size, &counts, &epochs)) {
1.195 + // Allocation was successful.
1.196 + // We can just free the old arrays; we're
1.197 + // not interested in preserving the contents
1.198 + if (_card_counts != NULL) os::free(_card_counts);
1.199 + if (_card_epochs != NULL) os::free(_card_epochs);
1.200 +
1.201 + // Cache the size of the arrays and the index that got us there.
1.202 + _n_card_counts = cache_size;
1.203 + _cache_size_index = cache_size_idx;
1.204 +
1.205 + _card_counts = counts;
1.206 + _card_epochs = epochs;
1.207 +
1.208 + // We successfully allocated/expanded the caches.
1.209 + return true;
1.210 + }
1.211 }
1.212 }
1.213 +
1.214 + // We did not successfully expand the caches.
1.215 + return false;
1.216 }
1.217
1.218 void ConcurrentG1Refine::clear_and_record_card_counts() {
1.219 @@ -415,10 +503,16 @@
1.220 #endif
1.221
1.222 if (_expand_card_counts) {
1.223 - expand_card_count_cache();
1.224 + int new_idx = _cache_size_index + 1;
1.225 +
1.226 + if (expand_card_count_cache(new_idx)) {
1.227 + // Allocation was successful and _n_card_counts has
1.228 + // been updated to the new size. We only need to clear
1.229 + // the epochs so we don't read a bogus epoch value
1.230 + // when inserting a card into the hot card cache.
1.231 + Copy::fill_to_bytes(&_card_epochs[0], _n_card_counts * sizeof(CardEpochCacheEntry));
1.232 + }
1.233 _expand_card_counts = false;
1.234 - // Only need to clear the epochs.
1.235 - Copy::fill_to_bytes(&_card_epochs[0], _n_card_counts * sizeof(CardEpochCacheEntry));
1.236 }
1.237
1.238 int this_epoch = (int) _n_periods;
