1.1 --- a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp Wed Dec 16 15:12:51 2009 -0800
1.2 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp Wed Dec 23 09:23:54 2009 -0800
1.3 @@ -62,18 +62,15 @@
1.4 // implementation, namely, the simple binary tree (splaying
1.5 // temporarily disabled).
1.6 switch (dictionaryChoice) {
1.7 - case FreeBlockDictionary::dictionaryBinaryTree:
1.8 - _dictionary = new BinaryTreeDictionary(mr);
1.9 - break;
1.10 case FreeBlockDictionary::dictionarySplayTree:
1.11 case FreeBlockDictionary::dictionarySkipList:
1.12 default:
1.13 warning("dictionaryChoice: selected option not understood; using"
1.14 " default BinaryTreeDictionary implementation instead.");
1.15 + case FreeBlockDictionary::dictionaryBinaryTree:
1.16 _dictionary = new BinaryTreeDictionary(mr);
1.17 break;
1.18 }
1.19 - splitBirth(mr.word_size());
1.20 assert(_dictionary != NULL, "CMS dictionary initialization");
1.21 // The indexed free lists are initially all empty and are lazily
1.22 // filled in on demand. Initialize the array elements to NULL.
1.23 @@ -388,6 +385,105 @@
1.24 return res;
1.25 }
1.26
1.27 +void CompactibleFreeListSpace::print_indexed_free_lists(outputStream* st)
1.28 +const {
1.29 + reportIndexedFreeListStatistics();
1.30 + gclog_or_tty->print_cr("Layout of Indexed Freelists");
1.31 + gclog_or_tty->print_cr("---------------------------");
1.32 + FreeList::print_labels_on(st, "size");
1.33 + for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) {
1.34 + _indexedFreeList[i].print_on(gclog_or_tty);
1.35 + for (FreeChunk* fc = _indexedFreeList[i].head(); fc != NULL;
1.36 + fc = fc->next()) {
1.37 + gclog_or_tty->print_cr("\t[" PTR_FORMAT "," PTR_FORMAT ") %s",
1.38 + fc, (HeapWord*)fc + i,
1.39 + fc->cantCoalesce() ? "\t CC" : "");
1.40 + }
1.41 + }
1.42 +}
1.43 +
1.44 +void CompactibleFreeListSpace::print_promo_info_blocks(outputStream* st)
1.45 +const {
1.46 + _promoInfo.print_on(st);
1.47 +}
1.48 +
1.49 +void CompactibleFreeListSpace::print_dictionary_free_lists(outputStream* st)
1.50 +const {
1.51 + _dictionary->reportStatistics();
1.52 + st->print_cr("Layout of Freelists in Tree");
1.53 + st->print_cr("---------------------------");
1.54 + _dictionary->print_free_lists(st);
1.55 +}
1.56 +
1.57 +class BlkPrintingClosure: public BlkClosure {
1.58 + const CMSCollector* _collector;
1.59 + const CompactibleFreeListSpace* _sp;
1.60 + const CMSBitMap* _live_bit_map;
1.61 + const bool _post_remark;
1.62 + outputStream* _st;
1.63 +public:
1.64 + BlkPrintingClosure(const CMSCollector* collector,
1.65 + const CompactibleFreeListSpace* sp,
1.66 + const CMSBitMap* live_bit_map,
1.67 + outputStream* st):
1.68 + _collector(collector),
1.69 + _sp(sp),
1.70 + _live_bit_map(live_bit_map),
1.71 + _post_remark(collector->abstract_state() > CMSCollector::FinalMarking),
1.72 + _st(st) { }
1.73 + size_t do_blk(HeapWord* addr);
1.74 +};
1.75 +
1.76 +size_t BlkPrintingClosure::do_blk(HeapWord* addr) {
1.77 + size_t sz = _sp->block_size_no_stall(addr, _collector);
1.78 + assert(sz != 0, "Should always be able to compute a size");
1.79 + if (_sp->block_is_obj(addr)) {
1.80 + const bool dead = _post_remark && !_live_bit_map->isMarked(addr);
1.81 + _st->print_cr(PTR_FORMAT ": %s object of size " SIZE_FORMAT "%s",
1.82 + addr,
1.83 + dead ? "dead" : "live",
1.84 + sz,
1.85 + (!dead && CMSPrintObjectsInDump) ? ":" : ".");
1.86 + if (CMSPrintObjectsInDump && !dead) {
1.87 + oop(addr)->print_on(_st);
1.88 + _st->print_cr("--------------------------------------");
1.89 + }
1.90 + } else { // free block
1.91 + _st->print_cr(PTR_FORMAT ": free block of size " SIZE_FORMAT "%s",
1.92 + addr, sz, CMSPrintChunksInDump ? ":" : ".");
1.93 + if (CMSPrintChunksInDump) {
1.94 + ((FreeChunk*)addr)->print_on(_st);
1.95 + _st->print_cr("--------------------------------------");
1.96 + }
1.97 + }
1.98 + return sz;
1.99 +}
1.100 +
1.101 +void CompactibleFreeListSpace::dump_at_safepoint_with_locks(CMSCollector* c,
1.102 + outputStream* st) {
1.103 + st->print_cr("\n=========================");
1.104 + st->print_cr("Block layout in CMS Heap:");
1.105 + st->print_cr("=========================");
1.106 + BlkPrintingClosure bpcl(c, this, c->markBitMap(), st);
1.107 + blk_iterate(&bpcl);
1.108 +
1.109 + st->print_cr("\n=======================================");
1.110 + st->print_cr("Order & Layout of Promotion Info Blocks");
1.111 + st->print_cr("=======================================");
1.112 + print_promo_info_blocks(st);
1.113 +
1.114 + st->print_cr("\n===========================");
1.115 + st->print_cr("Order of Indexed Free Lists");
1.116 + st->print_cr("=========================");
1.117 + print_indexed_free_lists(st);
1.118 +
1.119 + st->print_cr("\n=================================");
1.120 + st->print_cr("Order of Free Lists in Dictionary");
1.121 + st->print_cr("=================================");
1.122 + print_dictionary_free_lists(st);
1.123 +}
1.124 +
1.125 +
1.126 void CompactibleFreeListSpace::reportFreeListStatistics() const {
1.127 assert_lock_strong(&_freelistLock);
1.128 assert(PrintFLSStatistics != 0, "Reporting error");
1.129 @@ -449,37 +545,37 @@
1.130 if (prevEnd != NULL) {
1.131 // Resize the underlying block offset table.
1.132 _bt.resize(pointer_delta(value, bottom()));
1.133 - if (value <= prevEnd) {
1.134 - assert(value >= unallocated_block(), "New end is below unallocated block");
1.135 - } else {
1.136 - // Now, take this new chunk and add it to the free blocks.
1.137 - // Note that the BOT has not yet been updated for this block.
1.138 - size_t newFcSize = pointer_delta(value, prevEnd);
1.139 - // XXX This is REALLY UGLY and should be fixed up. XXX
1.140 - if (!_adaptive_freelists && _smallLinearAllocBlock._ptr == NULL) {
1.141 - // Mark the boundary of the new block in BOT
1.142 - _bt.mark_block(prevEnd, value);
1.143 - // put it all in the linAB
1.144 - if (ParallelGCThreads == 0) {
1.145 - _smallLinearAllocBlock._ptr = prevEnd;
1.146 - _smallLinearAllocBlock._word_size = newFcSize;
1.147 - repairLinearAllocBlock(&_smallLinearAllocBlock);
1.148 - } else { // ParallelGCThreads > 0
1.149 - MutexLockerEx x(parDictionaryAllocLock(),
1.150 - Mutex::_no_safepoint_check_flag);
1.151 - _smallLinearAllocBlock._ptr = prevEnd;
1.152 - _smallLinearAllocBlock._word_size = newFcSize;
1.153 - repairLinearAllocBlock(&_smallLinearAllocBlock);
1.154 + if (value <= prevEnd) {
1.155 + assert(value >= unallocated_block(), "New end is below unallocated block");
1.156 + } else {
1.157 + // Now, take this new chunk and add it to the free blocks.
1.158 + // Note that the BOT has not yet been updated for this block.
1.159 + size_t newFcSize = pointer_delta(value, prevEnd);
1.160 + // XXX This is REALLY UGLY and should be fixed up. XXX
1.161 + if (!_adaptive_freelists && _smallLinearAllocBlock._ptr == NULL) {
1.162 + // Mark the boundary of the new block in BOT
1.163 + _bt.mark_block(prevEnd, value);
1.164 + // put it all in the linAB
1.165 + if (ParallelGCThreads == 0) {
1.166 + _smallLinearAllocBlock._ptr = prevEnd;
1.167 + _smallLinearAllocBlock._word_size = newFcSize;
1.168 + repairLinearAllocBlock(&_smallLinearAllocBlock);
1.169 + } else { // ParallelGCThreads > 0
1.170 + MutexLockerEx x(parDictionaryAllocLock(),
1.171 + Mutex::_no_safepoint_check_flag);
1.172 + _smallLinearAllocBlock._ptr = prevEnd;
1.173 + _smallLinearAllocBlock._word_size = newFcSize;
1.174 + repairLinearAllocBlock(&_smallLinearAllocBlock);
1.175 + }
1.176 + // Births of chunks put into a LinAB are not recorded. Births
1.177 + // of chunks as they are allocated out of a LinAB are.
1.178 + } else {
1.179 + // Add the block to the free lists, if possible coalescing it
1.180 + // with the last free block, and update the BOT and census data.
1.181 + addChunkToFreeListsAtEndRecordingStats(prevEnd, newFcSize);
1.182 }
1.183 - // Births of chunks put into a LinAB are not recorded. Births
1.184 - // of chunks as they are allocated out of a LinAB are.
1.185 - } else {
1.186 - // Add the block to the free lists, if possible coalescing it
1.187 - // with the last free block, and update the BOT and census data.
1.188 - addChunkToFreeListsAtEndRecordingStats(prevEnd, newFcSize);
1.189 }
1.190 }
1.191 - }
1.192 }
1.193
1.194 class FreeListSpace_DCTOC : public Filtering_DCTOC {
1.195 @@ -732,7 +828,7 @@
1.196
1.197 void CompactibleFreeListSpace::object_iterate_mem(MemRegion mr,
1.198 UpwardsObjectClosure* cl) {
1.199 - assert_locked();
1.200 + assert_locked(freelistLock());
1.201 NOT_PRODUCT(verify_objects_initialized());
1.202 Space::object_iterate_mem(mr, cl);
1.203 }
1.204 @@ -1212,12 +1308,15 @@
1.205 void CompactibleFreeListSpace::assert_locked() const {
1.206 CMSLockVerifier::assert_locked(freelistLock(), parDictionaryAllocLock());
1.207 }
1.208 +
1.209 +void CompactibleFreeListSpace::assert_locked(const Mutex* lock) const {
1.210 + CMSLockVerifier::assert_locked(lock);
1.211 +}
1.212 #endif
1.213
1.214 FreeChunk* CompactibleFreeListSpace::allocateScratch(size_t size) {
1.215 // In the parallel case, the main thread holds the free list lock
1.216 // on behalf the parallel threads.
1.217 - assert_locked();
1.218 FreeChunk* fc;
1.219 {
1.220 // If GC is parallel, this might be called by several threads.
1.221 @@ -1298,17 +1397,18 @@
1.222 res = blk->_ptr;
1.223 _bt.allocated(res, blk->_word_size);
1.224 } else if (size + MinChunkSize <= blk->_refillSize) {
1.225 + size_t sz = blk->_word_size;
1.226 // Update _unallocated_block if the size is such that chunk would be
1.227 // returned to the indexed free list. All other chunks in the indexed
1.228 // free lists are allocated from the dictionary so that _unallocated_block
1.229 // has already been adjusted for them. Do it here so that the cost
1.230 // for all chunks added back to the indexed free lists.
1.231 - if (blk->_word_size < SmallForDictionary) {
1.232 - _bt.allocated(blk->_ptr, blk->_word_size);
1.233 + if (sz < SmallForDictionary) {
1.234 + _bt.allocated(blk->_ptr, sz);
1.235 }
1.236 // Return the chunk that isn't big enough, and then refill below.
1.237 - addChunkToFreeLists(blk->_ptr, blk->_word_size);
1.238 - _bt.verify_single_block(blk->_ptr, (blk->_ptr + blk->_word_size));
1.239 + addChunkToFreeLists(blk->_ptr, sz);
1.240 + splitBirth(sz);
1.241 // Don't keep statistics on adding back chunk from a LinAB.
1.242 } else {
1.243 // A refilled block would not satisfy the request.
1.244 @@ -1376,11 +1476,13 @@
1.245 res = getChunkFromIndexedFreeListHelper(size);
1.246 }
1.247 _bt.verify_not_unallocated((HeapWord*) res, size);
1.248 + assert(res == NULL || res->size() == size, "Incorrect block size");
1.249 return res;
1.250 }
1.251
1.252 FreeChunk*
1.253 -CompactibleFreeListSpace::getChunkFromIndexedFreeListHelper(size_t size) {
1.254 +CompactibleFreeListSpace::getChunkFromIndexedFreeListHelper(size_t size,
1.255 + bool replenish) {
1.256 assert_locked();
1.257 FreeChunk* fc = NULL;
1.258 if (size < SmallForDictionary) {
1.259 @@ -1398,54 +1500,66 @@
1.260 // and replenishing indexed lists from the small linAB.
1.261 //
1.262 FreeChunk* newFc = NULL;
1.263 - size_t replenish_size = CMSIndexedFreeListReplenish * size;
1.264 + const size_t replenish_size = CMSIndexedFreeListReplenish * size;
1.265 if (replenish_size < SmallForDictionary) {
1.266 // Do not replenish from an underpopulated size.
1.267 if (_indexedFreeList[replenish_size].surplus() > 0 &&
1.268 _indexedFreeList[replenish_size].head() != NULL) {
1.269 - newFc =
1.270 - _indexedFreeList[replenish_size].getChunkAtHead();
1.271 - } else {
1.272 + newFc = _indexedFreeList[replenish_size].getChunkAtHead();
1.273 + } else if (bestFitFirst()) {
1.274 newFc = bestFitSmall(replenish_size);
1.275 }
1.276 }
1.277 + if (newFc == NULL && replenish_size > size) {
1.278 + assert(CMSIndexedFreeListReplenish > 1, "ctl pt invariant");
1.279 + newFc = getChunkFromIndexedFreeListHelper(replenish_size, false);
1.280 + }
1.281 + // Note: The stats update re split-death of block obtained above
1.282 + // will be recorded below precisely when we know we are going to
1.283 + // be actually splitting it into more than one pieces below.
1.284 if (newFc != NULL) {
1.285 - splitDeath(replenish_size);
1.286 - } else if (replenish_size > size) {
1.287 - assert(CMSIndexedFreeListReplenish > 1, "ctl pt invariant");
1.288 - newFc =
1.289 - getChunkFromIndexedFreeListHelper(replenish_size);
1.290 - }
1.291 - if (newFc != NULL) {
1.292 - assert(newFc->size() == replenish_size, "Got wrong size");
1.293 - size_t i;
1.294 - FreeChunk *curFc, *nextFc;
1.295 - // carve up and link blocks 0, ..., CMSIndexedFreeListReplenish - 2
1.296 - // The last chunk is not added to the lists but is returned as the
1.297 - // free chunk.
1.298 - for (curFc = newFc, nextFc = (FreeChunk*)((HeapWord*)curFc + size),
1.299 - i = 0;
1.300 - i < (CMSIndexedFreeListReplenish - 1);
1.301 - curFc = nextFc, nextFc = (FreeChunk*)((HeapWord*)nextFc + size),
1.302 - i++) {
1.303 + if (replenish || CMSReplenishIntermediate) {
1.304 + // Replenish this list and return one block to caller.
1.305 + size_t i;
1.306 + FreeChunk *curFc, *nextFc;
1.307 + size_t num_blk = newFc->size() / size;
1.308 + assert(num_blk >= 1, "Smaller than requested?");
1.309 + assert(newFc->size() % size == 0, "Should be integral multiple of request");
1.310 + if (num_blk > 1) {
1.311 + // we are sure we will be splitting the block just obtained
1.312 + // into multiple pieces; record the split-death of the original
1.313 + splitDeath(replenish_size);
1.314 + }
1.315 + // carve up and link blocks 0, ..., num_blk - 2
1.316 + // The last chunk is not added to the lists but is returned as the
1.317 + // free chunk.
1.318 + for (curFc = newFc, nextFc = (FreeChunk*)((HeapWord*)curFc + size),
1.319 + i = 0;
1.320 + i < (num_blk - 1);
1.321 + curFc = nextFc, nextFc = (FreeChunk*)((HeapWord*)nextFc + size),
1.322 + i++) {
1.323 + curFc->setSize(size);
1.324 + // Don't record this as a return in order to try and
1.325 + // determine the "returns" from a GC.
1.326 + _bt.verify_not_unallocated((HeapWord*) fc, size);
1.327 + _indexedFreeList[size].returnChunkAtTail(curFc, false);
1.328 + _bt.mark_block((HeapWord*)curFc, size);
1.329 + splitBirth(size);
1.330 + // Don't record the initial population of the indexed list
1.331 + // as a split birth.
1.332 + }
1.333 +
1.334 + // check that the arithmetic was OK above
1.335 + assert((HeapWord*)nextFc == (HeapWord*)newFc + num_blk*size,
1.336 + "inconsistency in carving newFc");
1.337 curFc->setSize(size);
1.338 - // Don't record this as a return in order to try and
1.339 - // determine the "returns" from a GC.
1.340 - _bt.verify_not_unallocated((HeapWord*) fc, size);
1.341 - _indexedFreeList[size].returnChunkAtTail(curFc, false);
1.342 _bt.mark_block((HeapWord*)curFc, size);
1.343 splitBirth(size);
1.344 - // Don't record the initial population of the indexed list
1.345 - // as a split birth.
1.346 + fc = curFc;
1.347 + } else {
1.348 + // Return entire block to caller
1.349 + fc = newFc;
1.350 }
1.351 -
1.352 - // check that the arithmetic was OK above
1.353 - assert((HeapWord*)nextFc == (HeapWord*)newFc + replenish_size,
1.354 - "inconsistency in carving newFc");
1.355 - curFc->setSize(size);
1.356 - _bt.mark_block((HeapWord*)curFc, size);
1.357 - splitBirth(size);
1.358 - return curFc;
1.359 }
1.360 }
1.361 } else {
1.362 @@ -1453,7 +1567,7 @@
1.363 // replenish the indexed free list.
1.364 fc = getChunkFromDictionaryExact(size);
1.365 }
1.366 - assert(fc == NULL || fc->isFree(), "Should be returning a free chunk");
1.367 + // assert(fc == NULL || fc->isFree(), "Should be returning a free chunk");
1.368 return fc;
1.369 }
1.370
1.371 @@ -1512,6 +1626,11 @@
1.372 // adjust _unallocated_block downward, as necessary
1.373 _bt.freed((HeapWord*)chunk, size);
1.374 _dictionary->returnChunk(chunk);
1.375 +#ifndef PRODUCT
1.376 + if (CMSCollector::abstract_state() != CMSCollector::Sweeping) {
1.377 + TreeChunk::as_TreeChunk(chunk)->list()->verify_stats();
1.378 + }
1.379 +#endif // PRODUCT
1.380 }
1.381
1.382 void
1.383 @@ -1525,6 +1644,11 @@
1.384 } else {
1.385 _indexedFreeList[size].returnChunkAtHead(fc);
1.386 }
1.387 +#ifndef PRODUCT
1.388 + if (CMSCollector::abstract_state() != CMSCollector::Sweeping) {
1.389 + _indexedFreeList[size].verify_stats();
1.390 + }
1.391 +#endif // PRODUCT
1.392 }
1.393
1.394 // Add chunk to end of last block -- if it's the largest
1.395 @@ -1537,7 +1661,6 @@
1.396 HeapWord* chunk, size_t size) {
1.397 // check that the chunk does lie in this space!
1.398 assert(chunk != NULL && is_in_reserved(chunk), "Not in this space!");
1.399 - assert_locked();
1.400 // One of the parallel gc task threads may be here
1.401 // whilst others are allocating.
1.402 Mutex* lock = NULL;
1.403 @@ -1991,24 +2114,26 @@
1.404 return frag;
1.405 }
1.406
1.407 -#define CoalSurplusPercent 1.05
1.408 -#define SplitSurplusPercent 1.10
1.409 -
1.410 void CompactibleFreeListSpace::beginSweepFLCensus(
1.411 float inter_sweep_current,
1.412 - float inter_sweep_estimate) {
1.413 + float inter_sweep_estimate,
1.414 + float intra_sweep_estimate) {
1.415 assert_locked();
1.416 size_t i;
1.417 for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) {
1.418 FreeList* fl = &_indexedFreeList[i];
1.419 - fl->compute_desired(inter_sweep_current, inter_sweep_estimate);
1.420 - fl->set_coalDesired((ssize_t)((double)fl->desired() * CoalSurplusPercent));
1.421 + if (PrintFLSStatistics > 1) {
1.422 + gclog_or_tty->print("size[%d] : ", i);
1.423 + }
1.424 + fl->compute_desired(inter_sweep_current, inter_sweep_estimate, intra_sweep_estimate);
1.425 + fl->set_coalDesired((ssize_t)((double)fl->desired() * CMSSmallCoalSurplusPercent));
1.426 fl->set_beforeSweep(fl->count());
1.427 fl->set_bfrSurp(fl->surplus());
1.428 }
1.429 - _dictionary->beginSweepDictCensus(CoalSurplusPercent,
1.430 + _dictionary->beginSweepDictCensus(CMSLargeCoalSurplusPercent,
1.431 inter_sweep_current,
1.432 - inter_sweep_estimate);
1.433 + inter_sweep_estimate,
1.434 + intra_sweep_estimate);
1.435 }
1.436
1.437 void CompactibleFreeListSpace::setFLSurplus() {
1.438 @@ -2017,7 +2142,7 @@
1.439 for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) {
1.440 FreeList *fl = &_indexedFreeList[i];
1.441 fl->set_surplus(fl->count() -
1.442 - (ssize_t)((double)fl->desired() * SplitSurplusPercent));
1.443 + (ssize_t)((double)fl->desired() * CMSSmallSplitSurplusPercent));
1.444 }
1.445 }
1.446
1.447 @@ -2048,6 +2173,11 @@
1.448 }
1.449
1.450 void CompactibleFreeListSpace::endSweepFLCensus(size_t sweep_count) {
1.451 + if (PrintFLSStatistics > 0) {
1.452 + HeapWord* largestAddr = (HeapWord*) dictionary()->findLargestDict();
1.453 + gclog_or_tty->print_cr("CMS: Large block " PTR_FORMAT,
1.454 + largestAddr);
1.455 + }
1.456 setFLSurplus();
1.457 setFLHints();
1.458 if (PrintGC && PrintFLSCensus > 0) {
1.459 @@ -2055,7 +2185,7 @@
1.460 }
1.461 clearFLCensus();
1.462 assert_locked();
1.463 - _dictionary->endSweepDictCensus(SplitSurplusPercent);
1.464 + _dictionary->endSweepDictCensus(CMSLargeSplitSurplusPercent);
1.465 }
1.466
1.467 bool CompactibleFreeListSpace::coalOverPopulated(size_t size) {
1.468 @@ -2312,13 +2442,18 @@
1.469 }
1.470
1.471 void CompactibleFreeListSpace::verifyIndexedFreeList(size_t size) const {
1.472 - FreeChunk* fc = _indexedFreeList[size].head();
1.473 + FreeChunk* fc = _indexedFreeList[size].head();
1.474 + FreeChunk* tail = _indexedFreeList[size].tail();
1.475 + size_t num = _indexedFreeList[size].count();
1.476 + size_t n = 0;
1.477 guarantee((size % 2 == 0) || fc == NULL, "Odd slots should be empty");
1.478 - for (; fc != NULL; fc = fc->next()) {
1.479 + for (; fc != NULL; fc = fc->next(), n++) {
1.480 guarantee(fc->size() == size, "Size inconsistency");
1.481 guarantee(fc->isFree(), "!free?");
1.482 guarantee(fc->next() == NULL || fc->next()->prev() == fc, "Broken list");
1.483 + guarantee((fc->next() == NULL) == (fc == tail), "Incorrect tail");
1.484 }
1.485 + guarantee(n == num, "Incorrect count");
1.486 }
1.487
1.488 #ifndef PRODUCT
1.489 @@ -2516,11 +2651,41 @@
1.490 _tracking = true;
1.491 }
1.492
1.493 -void PromotionInfo::stopTrackingPromotions() {
1.494 +#define CMSPrintPromoBlockInfo 1
1.495 +
1.496 +void PromotionInfo::stopTrackingPromotions(uint worker_id) {
1.497 assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex,
1.498 "spooling inconsistency?");
1.499 _firstIndex = _nextIndex = 1;
1.500 _tracking = false;
1.501 + if (CMSPrintPromoBlockInfo > 1) {
1.502 + print_statistics(worker_id);
1.503 + }
1.504 +}
1.505 +
1.506 +void PromotionInfo::print_statistics(uint worker_id) const {
1.507 + assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex,
1.508 + "Else will undercount");
1.509 + assert(CMSPrintPromoBlockInfo > 0, "Else unnecessary call");
1.510 + // Count the number of blocks and slots in the free pool
1.511 + size_t slots = 0;
1.512 + size_t blocks = 0;
1.513 + for (SpoolBlock* cur_spool = _spareSpool;
1.514 + cur_spool != NULL;
1.515 + cur_spool = cur_spool->nextSpoolBlock) {
1.516 + // the first entry is just a self-pointer; indices 1 through
1.517 + // bufferSize - 1 are occupied (thus, bufferSize - 1 slots).
1.518 + guarantee((void*)cur_spool->displacedHdr == (void*)&cur_spool->displacedHdr,
1.519 + "first entry of displacedHdr should be self-referential");
1.520 + slots += cur_spool->bufferSize - 1;
1.521 + blocks++;
1.522 + }
1.523 + if (_spoolHead != NULL) {
1.524 + slots += _spoolHead->bufferSize - 1;
1.525 + blocks++;
1.526 + }
1.527 + gclog_or_tty->print_cr(" [worker %d] promo_blocks = %d, promo_slots = %d ",
1.528 + worker_id, blocks, slots);
1.529 }
1.530
1.531 // When _spoolTail is not NULL, then the slot <_spoolTail, _nextIndex>
1.532 @@ -2584,15 +2749,84 @@
1.533 guarantee(numDisplacedHdrs == numObjsWithDisplacedHdrs, "Displaced hdr count");
1.534 }
1.535
1.536 +void PromotionInfo::print_on(outputStream* st) const {
1.537 + SpoolBlock* curSpool = NULL;
1.538 + size_t i = 0;
1.539 + st->print_cr("start & end indices: [" SIZE_FORMAT ", " SIZE_FORMAT ")",
1.540 + _firstIndex, _nextIndex);
1.541 + for (curSpool = _spoolHead; curSpool != _spoolTail && curSpool != NULL;
1.542 + curSpool = curSpool->nextSpoolBlock) {
1.543 + curSpool->print_on(st);
1.544 + st->print_cr(" active ");
1.545 + i++;
1.546 + }
1.547 + for (curSpool = _spoolTail; curSpool != NULL;
1.548 + curSpool = curSpool->nextSpoolBlock) {
1.549 + curSpool->print_on(st);
1.550 + st->print_cr(" inactive ");
1.551 + i++;
1.552 + }
1.553 + for (curSpool = _spareSpool; curSpool != NULL;
1.554 + curSpool = curSpool->nextSpoolBlock) {
1.555 + curSpool->print_on(st);
1.556 + st->print_cr(" free ");
1.557 + i++;
1.558 + }
1.559 + st->print_cr(SIZE_FORMAT " header spooling blocks", i);
1.560 +}
1.561 +
1.562 +void SpoolBlock::print_on(outputStream* st) const {
1.563 + st->print("[" PTR_FORMAT "," PTR_FORMAT "), " SIZE_FORMAT " HeapWords -> " PTR_FORMAT,
1.564 + this, (HeapWord*)displacedHdr + bufferSize,
1.565 + bufferSize, nextSpoolBlock);
1.566 +}
1.567 +
1.568 +///////////////////////////////////////////////////////////////////////////
1.569 +// CFLS_LAB
1.570 +///////////////////////////////////////////////////////////////////////////
1.571 +
1.572 +#define VECTOR_257(x) \
1.573 + /* 1 2 3 4 5 6 7 8 9 1x 11 12 13 14 15 16 17 18 19 2x 21 22 23 24 25 26 27 28 29 3x 31 32 */ \
1.574 + { x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \
1.575 + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \
1.576 + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \
1.577 + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \
1.578 + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \
1.579 + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \
1.580 + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \
1.581 + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \
1.582 + x }
1.583 +
1.584 +// Initialize with default setting of CMSParPromoteBlocksToClaim, _not_
1.585 +// OldPLABSize, whose static default is different; if overridden at the
1.586 +// command-line, this will get reinitialized via a call to
1.587 +// modify_initialization() below.
1.588 +AdaptiveWeightedAverage CFLS_LAB::_blocks_to_claim[] =
1.589 + VECTOR_257(AdaptiveWeightedAverage(OldPLABWeight, (float)CMSParPromoteBlocksToClaim));
1.590 +size_t CFLS_LAB::_global_num_blocks[] = VECTOR_257(0);
1.591 +int CFLS_LAB::_global_num_workers[] = VECTOR_257(0);
1.592
1.593 CFLS_LAB::CFLS_LAB(CompactibleFreeListSpace* cfls) :
1.594 _cfls(cfls)
1.595 {
1.596 - _blocks_to_claim = CMSParPromoteBlocksToClaim;
1.597 + assert(CompactibleFreeListSpace::IndexSetSize == 257, "Modify VECTOR_257() macro above");
1.598 for (size_t i = CompactibleFreeListSpace::IndexSetStart;
1.599 i < CompactibleFreeListSpace::IndexSetSize;
1.600 i += CompactibleFreeListSpace::IndexSetStride) {
1.601 _indexedFreeList[i].set_size(i);
1.602 + _num_blocks[i] = 0;
1.603 + }
1.604 +}
1.605 +
1.606 +static bool _CFLS_LAB_modified = false;
1.607 +
1.608 +void CFLS_LAB::modify_initialization(size_t n, unsigned wt) {
1.609 + assert(!_CFLS_LAB_modified, "Call only once");
1.610 + _CFLS_LAB_modified = true;
1.611 + for (size_t i = CompactibleFreeListSpace::IndexSetStart;
1.612 + i < CompactibleFreeListSpace::IndexSetSize;
1.613 + i += CompactibleFreeListSpace::IndexSetStride) {
1.614 + _blocks_to_claim[i].modify(n, wt, true /* force */);
1.615 }
1.616 }
1.617
1.618 @@ -2607,11 +2841,9 @@
1.619 if (res == NULL) return NULL;
1.620 } else {
1.621 FreeList* fl = &_indexedFreeList[word_sz];
1.622 - bool filled = false; //TRAP
1.623 if (fl->count() == 0) {
1.624 - bool filled = true; //TRAP
1.625 // Attempt to refill this local free list.
1.626 - _cfls->par_get_chunk_of_blocks(word_sz, _blocks_to_claim, fl);
1.627 + get_from_global_pool(word_sz, fl);
1.628 // If it didn't work, give up.
1.629 if (fl->count() == 0) return NULL;
1.630 }
1.631 @@ -2626,80 +2858,190 @@
1.632 return (HeapWord*)res;
1.633 }
1.634
1.635 -void CFLS_LAB::retire() {
1.636 - for (size_t i = CompactibleFreeListSpace::IndexSetStart;
1.637 +// Get a chunk of blocks of the right size and update related
1.638 +// book-keeping stats
1.639 +void CFLS_LAB::get_from_global_pool(size_t word_sz, FreeList* fl) {
1.640 + // Get the #blocks we want to claim
1.641 + size_t n_blks = (size_t)_blocks_to_claim[word_sz].average();
1.642 + assert(n_blks > 0, "Error");
1.643 + assert(ResizePLAB || n_blks == OldPLABSize, "Error");
1.644 + // In some cases, when the application has a phase change,
1.645 + // there may be a sudden and sharp shift in the object survival
1.646 + // profile, and updating the counts at the end of a scavenge
1.647 + // may not be quick enough, giving rise to large scavenge pauses
1.648 + // during these phase changes. It is beneficial to detect such
1.649 + // changes on-the-fly during a scavenge and avoid such a phase-change
1.650 + // pothole. The following code is a heuristic attempt to do that.
1.651 + // It is protected by a product flag until we have gained
1.652 + // enough experience with this heuristic and fine-tuned its behaviour.
1.653 + // WARNING: This might increase fragmentation if we overreact to
1.654 + // small spikes, so some kind of historical smoothing based on
1.655 + // previous experience with the greater reactivity might be useful.
1.656 + // Lacking sufficient experience, CMSOldPLABResizeQuicker is disabled by
1.657 + // default.
1.658 + if (ResizeOldPLAB && CMSOldPLABResizeQuicker) {
1.659 + size_t multiple = _num_blocks[word_sz]/(CMSOldPLABToleranceFactor*CMSOldPLABNumRefills*n_blks);
1.660 + n_blks += CMSOldPLABReactivityFactor*multiple*n_blks;
1.661 + n_blks = MIN2(n_blks, CMSOldPLABMax);
1.662 + }
1.663 + assert(n_blks > 0, "Error");
1.664 + _cfls->par_get_chunk_of_blocks(word_sz, n_blks, fl);
1.665 + // Update stats table entry for this block size
1.666 + _num_blocks[word_sz] += fl->count();
1.667 +}
1.668 +
1.669 +void CFLS_LAB::compute_desired_plab_size() {
1.670 + for (size_t i = CompactibleFreeListSpace::IndexSetStart;
1.671 i < CompactibleFreeListSpace::IndexSetSize;
1.672 i += CompactibleFreeListSpace::IndexSetStride) {
1.673 - if (_indexedFreeList[i].count() > 0) {
1.674 - MutexLockerEx x(_cfls->_indexedFreeListParLocks[i],
1.675 - Mutex::_no_safepoint_check_flag);
1.676 - _cfls->_indexedFreeList[i].prepend(&_indexedFreeList[i]);
1.677 - // Reset this list.
1.678 - _indexedFreeList[i] = FreeList();
1.679 - _indexedFreeList[i].set_size(i);
1.680 + assert((_global_num_workers[i] == 0) == (_global_num_blocks[i] == 0),
1.681 + "Counter inconsistency");
1.682 + if (_global_num_workers[i] > 0) {
1.683 + // Need to smooth wrt historical average
1.684 + if (ResizeOldPLAB) {
1.685 + _blocks_to_claim[i].sample(
1.686 + MAX2((size_t)CMSOldPLABMin,
1.687 + MIN2((size_t)CMSOldPLABMax,
1.688 + _global_num_blocks[i]/(_global_num_workers[i]*CMSOldPLABNumRefills))));
1.689 + }
1.690 + // Reset counters for next round
1.691 + _global_num_workers[i] = 0;
1.692 + _global_num_blocks[i] = 0;
1.693 + if (PrintOldPLAB) {
1.694 + gclog_or_tty->print_cr("[%d]: %d", i, (size_t)_blocks_to_claim[i].average());
1.695 + }
1.696 }
1.697 }
1.698 }
1.699
1.700 -void
1.701 -CompactibleFreeListSpace::
1.702 -par_get_chunk_of_blocks(size_t word_sz, size_t n, FreeList* fl) {
1.703 +void CFLS_LAB::retire(int tid) {
1.704 + // We run this single threaded with the world stopped;
1.705 + // so no need for locks and such.
1.706 +#define CFLS_LAB_PARALLEL_ACCESS 0
1.707 + NOT_PRODUCT(Thread* t = Thread::current();)
1.708 + assert(Thread::current()->is_VM_thread(), "Error");
1.709 + assert(CompactibleFreeListSpace::IndexSetStart == CompactibleFreeListSpace::IndexSetStride,
1.710 + "Will access to uninitialized slot below");
1.711 +#if CFLS_LAB_PARALLEL_ACCESS
1.712 + for (size_t i = CompactibleFreeListSpace::IndexSetSize - 1;
1.713 + i > 0;
1.714 + i -= CompactibleFreeListSpace::IndexSetStride) {
1.715 +#else // CFLS_LAB_PARALLEL_ACCESS
1.716 + for (size_t i = CompactibleFreeListSpace::IndexSetStart;
1.717 + i < CompactibleFreeListSpace::IndexSetSize;
1.718 + i += CompactibleFreeListSpace::IndexSetStride) {
1.719 +#endif // !CFLS_LAB_PARALLEL_ACCESS
1.720 + assert(_num_blocks[i] >= (size_t)_indexedFreeList[i].count(),
1.721 + "Can't retire more than what we obtained");
1.722 + if (_num_blocks[i] > 0) {
1.723 + size_t num_retire = _indexedFreeList[i].count();
1.724 + assert(_num_blocks[i] > num_retire, "Should have used at least one");
1.725 + {
1.726 +#if CFLS_LAB_PARALLEL_ACCESS
1.727 + MutexLockerEx x(_cfls->_indexedFreeListParLocks[i],
1.728 + Mutex::_no_safepoint_check_flag);
1.729 +#endif // CFLS_LAB_PARALLEL_ACCESS
1.730 + // Update globals stats for num_blocks used
1.731 + _global_num_blocks[i] += (_num_blocks[i] - num_retire);
1.732 + _global_num_workers[i]++;
1.733 + assert(_global_num_workers[i] <= (ssize_t)ParallelGCThreads, "Too big");
1.734 + if (num_retire > 0) {
1.735 + _cfls->_indexedFreeList[i].prepend(&_indexedFreeList[i]);
1.736 + // Reset this list.
1.737 + _indexedFreeList[i] = FreeList();
1.738 + _indexedFreeList[i].set_size(i);
1.739 + }
1.740 + }
1.741 + if (PrintOldPLAB) {
1.742 + gclog_or_tty->print_cr("%d[%d]: %d/%d/%d",
1.743 + tid, i, num_retire, _num_blocks[i], (size_t)_blocks_to_claim[i].average());
1.744 + }
1.745 + // Reset stats for next round
1.746 + _num_blocks[i] = 0;
1.747 + }
1.748 + }
1.749 +}
1.750 +
1.751 +void CompactibleFreeListSpace:: par_get_chunk_of_blocks(size_t word_sz, size_t n, FreeList* fl) {
1.752 assert(fl->count() == 0, "Precondition.");
1.753 assert(word_sz < CompactibleFreeListSpace::IndexSetSize,
1.754 "Precondition");
1.755
1.756 - // We'll try all multiples of word_sz in the indexed set (starting with
1.757 - // word_sz itself), then try getting a big chunk and splitting it.
1.758 - int k = 1;
1.759 - size_t cur_sz = k * word_sz;
1.760 - bool found = false;
1.761 - while (cur_sz < CompactibleFreeListSpace::IndexSetSize && k == 1) {
1.762 - FreeList* gfl = &_indexedFreeList[cur_sz];
1.763 - FreeList fl_for_cur_sz; // Empty.
1.764 - fl_for_cur_sz.set_size(cur_sz);
1.765 - {
1.766 - MutexLockerEx x(_indexedFreeListParLocks[cur_sz],
1.767 - Mutex::_no_safepoint_check_flag);
1.768 - if (gfl->count() != 0) {
1.769 - size_t nn = MAX2(n/k, (size_t)1);
1.770 - gfl->getFirstNChunksFromList(nn, &fl_for_cur_sz);
1.771 - found = true;
1.772 + // We'll try all multiples of word_sz in the indexed set, starting with
1.773 + // word_sz itself and, if CMSSplitIndexedFreeListBlocks, try larger multiples,
1.774 + // then try getting a big chunk and splitting it.
1.775 + {
1.776 + bool found;
1.777 + int k;
1.778 + size_t cur_sz;
1.779 + for (k = 1, cur_sz = k * word_sz, found = false;
1.780 + (cur_sz < CompactibleFreeListSpace::IndexSetSize) &&
1.781 + (CMSSplitIndexedFreeListBlocks || k <= 1);
1.782 + k++, cur_sz = k * word_sz) {
1.783 + FreeList* gfl = &_indexedFreeList[cur_sz];
1.784 + FreeList fl_for_cur_sz; // Empty.
1.785 + fl_for_cur_sz.set_size(cur_sz);
1.786 + {
1.787 + MutexLockerEx x(_indexedFreeListParLocks[cur_sz],
1.788 + Mutex::_no_safepoint_check_flag);
1.789 + if (gfl->count() != 0) {
1.790 + // nn is the number of chunks of size cur_sz that
1.791 + // we'd need to split k-ways each, in order to create
1.792 + // "n" chunks of size word_sz each.
1.793 + const size_t nn = MAX2(n/k, (size_t)1);
1.794 + gfl->getFirstNChunksFromList(nn, &fl_for_cur_sz);
1.795 + found = true;
1.796 + if (k > 1) {
1.797 + // Update split death stats for the cur_sz-size blocks list:
1.798 + // we increment the split death count by the number of blocks
1.799 + // we just took from the cur_sz-size blocks list and which
1.800 + // we will be splitting below.
1.801 + ssize_t deaths = _indexedFreeList[cur_sz].splitDeaths() +
1.802 + fl_for_cur_sz.count();
1.803 + _indexedFreeList[cur_sz].set_splitDeaths(deaths);
1.804 + }
1.805 + }
1.806 + }
1.807 + // Now transfer fl_for_cur_sz to fl. Common case, we hope, is k = 1.
1.808 + if (found) {
1.809 + if (k == 1) {
1.810 + fl->prepend(&fl_for_cur_sz);
1.811 + } else {
1.812 + // Divide each block on fl_for_cur_sz up k ways.
1.813 + FreeChunk* fc;
1.814 + while ((fc = fl_for_cur_sz.getChunkAtHead()) != NULL) {
1.815 + // Must do this in reverse order, so that anybody attempting to
1.816 + // access the main chunk sees it as a single free block until we
1.817 + // change it.
1.818 + size_t fc_size = fc->size();
1.819 + for (int i = k-1; i >= 0; i--) {
1.820 + FreeChunk* ffc = (FreeChunk*)((HeapWord*)fc + i * word_sz);
1.821 + ffc->setSize(word_sz);
1.822 + ffc->linkNext(NULL);
1.823 + ffc->linkPrev(NULL); // Mark as a free block for other (parallel) GC threads.
1.824 + // Above must occur before BOT is updated below.
1.825 + // splitting from the right, fc_size == (k - i + 1) * wordsize
1.826 + _bt.mark_block((HeapWord*)ffc, word_sz);
1.827 + fc_size -= word_sz;
1.828 + _bt.verify_not_unallocated((HeapWord*)ffc, ffc->size());
1.829 + _bt.verify_single_block((HeapWord*)fc, fc_size);
1.830 + _bt.verify_single_block((HeapWord*)ffc, ffc->size());
1.831 + // Push this on "fl".
1.832 + fl->returnChunkAtHead(ffc);
1.833 + }
1.834 + // TRAP
1.835 + assert(fl->tail()->next() == NULL, "List invariant.");
1.836 + }
1.837 + }
1.838 + // Update birth stats for this block size.
1.839 + size_t num = fl->count();
1.840 + MutexLockerEx x(_indexedFreeListParLocks[word_sz],
1.841 + Mutex::_no_safepoint_check_flag);
1.842 + ssize_t births = _indexedFreeList[word_sz].splitBirths() + num;
1.843 + _indexedFreeList[word_sz].set_splitBirths(births);
1.844 + return;
1.845 }
1.846 }
1.847 - // Now transfer fl_for_cur_sz to fl. Common case, we hope, is k = 1.
1.848 - if (found) {
1.849 - if (k == 1) {
1.850 - fl->prepend(&fl_for_cur_sz);
1.851 - } else {
1.852 - // Divide each block on fl_for_cur_sz up k ways.
1.853 - FreeChunk* fc;
1.854 - while ((fc = fl_for_cur_sz.getChunkAtHead()) != NULL) {
1.855 - // Must do this in reverse order, so that anybody attempting to
1.856 - // access the main chunk sees it as a single free block until we
1.857 - // change it.
1.858 - size_t fc_size = fc->size();
1.859 - for (int i = k-1; i >= 0; i--) {
1.860 - FreeChunk* ffc = (FreeChunk*)((HeapWord*)fc + i * word_sz);
1.861 - ffc->setSize(word_sz);
1.862 - ffc->linkNext(NULL);
1.863 - ffc->linkPrev(NULL); // Mark as a free block for other (parallel) GC threads.
1.864 - // Above must occur before BOT is updated below.
1.865 - // splitting from the right, fc_size == (k - i + 1) * wordsize
1.866 - _bt.mark_block((HeapWord*)ffc, word_sz);
1.867 - fc_size -= word_sz;
1.868 - _bt.verify_not_unallocated((HeapWord*)ffc, ffc->size());
1.869 - _bt.verify_single_block((HeapWord*)fc, fc_size);
1.870 - _bt.verify_single_block((HeapWord*)ffc, ffc->size());
1.871 - // Push this on "fl".
1.872 - fl->returnChunkAtHead(ffc);
1.873 - }
1.874 - // TRAP
1.875 - assert(fl->tail()->next() == NULL, "List invariant.");
1.876 - }
1.877 - }
1.878 - return;
1.879 - }
1.880 - k++; cur_sz = k * word_sz;
1.881 }
1.882 // Otherwise, we'll split a block from the dictionary.
1.883 FreeChunk* fc = NULL;
1.884 @@ -2723,17 +3065,20 @@
1.885 }
1.886 }
1.887 if (fc == NULL) return;
1.888 + assert((ssize_t)n >= 1, "Control point invariant");
1.889 // Otherwise, split up that block.
1.890 - size_t nn = fc->size() / word_sz;
1.891 + const size_t nn = fc->size() / word_sz;
1.892 n = MIN2(nn, n);
1.893 + assert((ssize_t)n >= 1, "Control point invariant");
1.894 rem = fc->size() - n * word_sz;
1.895 // If there is a remainder, and it's too small, allocate one fewer.
1.896 if (rem > 0 && rem < MinChunkSize) {
1.897 n--; rem += word_sz;
1.898 }
1.899 + assert((ssize_t)n >= 1, "Control point invariant");
1.900 // First return the remainder, if any.
1.901 // Note that we hold the lock until we decide if we're going to give
1.902 - // back the remainder to the dictionary, since a contending allocator
1.903 + // back the remainder to the dictionary, since a concurrent allocation
1.904 // may otherwise see the heap as empty. (We're willing to take that
1.905 // hit if the block is a small block.)
1.906 if (rem > 0) {
1.907 @@ -2743,18 +3088,16 @@
1.908 rem_fc->linkNext(NULL);
1.909 rem_fc->linkPrev(NULL); // Mark as a free block for other (parallel) GC threads.
1.910 // Above must occur before BOT is updated below.
1.911 + assert((ssize_t)n > 0 && prefix_size > 0 && rem_fc > fc, "Error");
1.912 _bt.split_block((HeapWord*)fc, fc->size(), prefix_size);
1.913 if (rem >= IndexSetSize) {
1.914 returnChunkToDictionary(rem_fc);
1.915 - dictionary()->dictCensusUpdate(fc->size(),
1.916 - true /*split*/,
1.917 - true /*birth*/);
1.918 + dictionary()->dictCensusUpdate(rem, true /*split*/, true /*birth*/);
1.919 rem_fc = NULL;
1.920 }
1.921 // Otherwise, return it to the small list below.
1.922 }
1.923 }
1.924 - //
1.925 if (rem_fc != NULL) {
1.926 MutexLockerEx x(_indexedFreeListParLocks[rem],
1.927 Mutex::_no_safepoint_check_flag);
1.928 @@ -2762,7 +3105,7 @@
1.929 _indexedFreeList[rem].returnChunkAtHead(rem_fc);
1.930 smallSplitBirth(rem);
1.931 }
1.932 -
1.933 + assert((ssize_t)n > 0 && fc != NULL, "Consistency");
1.934 // Now do the splitting up.
1.935 // Must do this in reverse order, so that anybody attempting to
1.936 // access the main chunk sees it as a single free block until we
1.937 @@ -2792,13 +3135,15 @@
1.938 _bt.verify_single_block((HeapWord*)fc, fc->size());
1.939 fl->returnChunkAtHead(fc);
1.940
1.941 + assert((ssize_t)n > 0 && (ssize_t)n == fl->count(), "Incorrect number of blocks");
1.942 {
1.943 + // Update the stats for this block size.
1.944 MutexLockerEx x(_indexedFreeListParLocks[word_sz],
1.945 Mutex::_no_safepoint_check_flag);
1.946 - ssize_t new_births = _indexedFreeList[word_sz].splitBirths() + n;
1.947 - _indexedFreeList[word_sz].set_splitBirths(new_births);
1.948 - ssize_t new_surplus = _indexedFreeList[word_sz].surplus() + n;
1.949 - _indexedFreeList[word_sz].set_surplus(new_surplus);
1.950 + const ssize_t births = _indexedFreeList[word_sz].splitBirths() + n;
1.951 + _indexedFreeList[word_sz].set_splitBirths(births);
1.952 + // ssize_t new_surplus = _indexedFreeList[word_sz].surplus() + n;
1.953 + // _indexedFreeList[word_sz].set_surplus(new_surplus);
1.954 }
1.955
1.956 // TRAP
