1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/promotionInfo.cpp Wed Apr 27 01:25:04 2016 +0800
1.3 @@ -0,0 +1,365 @@
1.4 +/*
1.5 + * Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved.
1.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
1.7 + *
1.8 + * This code is free software; you can redistribute it and/or modify it
1.9 + * under the terms of the GNU General Public License version 2 only, as
1.10 + * published by the Free Software Foundation.
1.11 + *
1.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
1.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1.14 + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
1.15 + * version 2 for more details (a copy is included in the LICENSE file that
1.16 + * accompanied this code).
1.17 + *
1.18 + * You should have received a copy of the GNU General Public License version
1.19 + * 2 along with this work; if not, write to the Free Software Foundation,
1.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
1.21 + *
1.22 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
1.23 + * or visit www.oracle.com if you need additional information or have any
1.24 + * questions.
1.25 + *
1.26 + */
1.27 +
1.28 +#include "precompiled.hpp"
1.29 +#include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp"
1.30 +#include "gc_implementation/concurrentMarkSweep/promotionInfo.hpp"
1.31 +#include "oops/markOop.inline.hpp"
1.32 +#include "oops/oop.inline.hpp"
1.33 +
1.34 +PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
1.35 +
1.36 +/////////////////////////////////////////////////////////////////////////
1.37 +//// PromotionInfo
1.38 +/////////////////////////////////////////////////////////////////////////
1.39 +
1.40 +
1.41 +//////////////////////////////////////////////////////////////////////////////
1.42 +// We go over the list of promoted objects, removing each from the list,
1.43 +// and applying the closure (this may, in turn, add more elements to
1.44 +// the tail of the promoted list, and these newly added objects will
1.45 +// also be processed) until the list is empty.
1.46 +// To aid verification and debugging, in the non-product builds
1.47 +// we actually forward _promoHead each time we process a promoted oop.
1.48 +// Note that this is not necessary in general (i.e. when we don't need to
1.49 +// call PromotionInfo::verify()) because oop_iterate can only add to the
1.50 +// end of _promoTail, and never needs to look at _promoHead.
1.51 +
1.52 +#define PROMOTED_OOPS_ITERATE_DEFN(OopClosureType, nv_suffix) \
1.53 + \
1.54 +void PromotionInfo::promoted_oops_iterate##nv_suffix(OopClosureType* cl) { \
1.55 + NOT_PRODUCT(verify()); \
1.56 + PromotedObject *curObj, *nextObj; \
1.57 + for (curObj = _promoHead; curObj != NULL; curObj = nextObj) { \
1.58 + if ((nextObj = curObj->next()) == NULL) { \
1.59 + /* protect ourselves against additions due to closure application \
1.60 + below by resetting the list. */ \
1.61 + assert(_promoTail == curObj, "Should have been the tail"); \
1.62 + _promoHead = _promoTail = NULL; \
1.63 + } \
1.64 + if (curObj->hasDisplacedMark()) { \
1.65 + /* restore displaced header */ \
1.66 + oop(curObj)->set_mark(nextDisplacedHeader()); \
1.67 + } else { \
1.68 + /* restore prototypical header */ \
1.69 + oop(curObj)->init_mark(); \
1.70 + } \
1.71 + /* The "promoted_mark" should now not be set */ \
1.72 + assert(!curObj->hasPromotedMark(), \
1.73 + "Should have been cleared by restoring displaced mark-word"); \
1.74 + NOT_PRODUCT(_promoHead = nextObj); \
1.75 + if (cl != NULL) oop(curObj)->oop_iterate(cl); \
1.76 + if (nextObj == NULL) { /* start at head of list reset above */ \
1.77 + nextObj = _promoHead; \
1.78 + } \
1.79 + } \
1.80 + assert(noPromotions(), "post-condition violation"); \
1.81 + assert(_promoHead == NULL && _promoTail == NULL, "emptied promoted list");\
1.82 + assert(_spoolHead == _spoolTail, "emptied spooling buffers"); \
1.83 + assert(_firstIndex == _nextIndex, "empty buffer"); \
1.84 +}
1.85 +
1.86 +// This should have been ALL_SINCE_...() just like the others,
1.87 +// but, because the body of the method above is somehwat longer,
1.88 +// the MSVC compiler cannot cope; as a workaround, we split the
1.89 +// macro into its 3 constituent parts below (see original macro
1.90 +// definition in specializedOopClosures.hpp).
1.91 +SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG(PROMOTED_OOPS_ITERATE_DEFN)
1.92 +PROMOTED_OOPS_ITERATE_DEFN(OopsInGenClosure,_v)
1.93 +
1.94 +
1.95 +// Return the next displaced header, incrementing the pointer and
1.96 +// recycling spool area as necessary.
1.97 +markOop PromotionInfo::nextDisplacedHeader() {
1.98 + assert(_spoolHead != NULL, "promotionInfo inconsistency");
1.99 + assert(_spoolHead != _spoolTail || _firstIndex < _nextIndex,
1.100 + "Empty spool space: no displaced header can be fetched");
1.101 + assert(_spoolHead->bufferSize > _firstIndex, "Off by one error at head?");
1.102 + markOop hdr = _spoolHead->displacedHdr[_firstIndex];
1.103 + // Spool forward
1.104 + if (++_firstIndex == _spoolHead->bufferSize) { // last location in this block
1.105 + // forward to next block, recycling this block into spare spool buffer
1.106 + SpoolBlock* tmp = _spoolHead->nextSpoolBlock;
1.107 + assert(_spoolHead != _spoolTail, "Spooling storage mix-up");
1.108 + _spoolHead->nextSpoolBlock = _spareSpool;
1.109 + _spareSpool = _spoolHead;
1.110 + _spoolHead = tmp;
1.111 + _firstIndex = 1;
1.112 + NOT_PRODUCT(
1.113 + if (_spoolHead == NULL) { // all buffers fully consumed
1.114 + assert(_spoolTail == NULL && _nextIndex == 1,
1.115 + "spool buffers processing inconsistency");
1.116 + }
1.117 + )
1.118 + }
1.119 + return hdr;
1.120 +}
1.121 +
1.122 +void PromotionInfo::track(PromotedObject* trackOop) {
1.123 + track(trackOop, oop(trackOop)->klass());
1.124 +}
1.125 +
1.126 +void PromotionInfo::track(PromotedObject* trackOop, Klass* klassOfOop) {
1.127 + // make a copy of header as it may need to be spooled
1.128 + markOop mark = oop(trackOop)->mark();
1.129 + trackOop->clear_next();
1.130 + if (mark->must_be_preserved_for_cms_scavenge(klassOfOop)) {
1.131 + // save non-prototypical header, and mark oop
1.132 + saveDisplacedHeader(mark);
1.133 + trackOop->setDisplacedMark();
1.134 + } else {
1.135 + // we'd like to assert something like the following:
1.136 + // assert(mark == markOopDesc::prototype(), "consistency check");
1.137 + // ... but the above won't work because the age bits have not (yet) been
1.138 + // cleared. The remainder of the check would be identical to the
1.139 + // condition checked in must_be_preserved() above, so we don't really
1.140 + // have anything useful to check here!
1.141 + }
1.142 + if (_promoTail != NULL) {
1.143 + assert(_promoHead != NULL, "List consistency");
1.144 + _promoTail->setNext(trackOop);
1.145 + _promoTail = trackOop;
1.146 + } else {
1.147 + assert(_promoHead == NULL, "List consistency");
1.148 + _promoHead = _promoTail = trackOop;
1.149 + }
1.150 + // Mask as newly promoted, so we can skip over such objects
1.151 + // when scanning dirty cards
1.152 + assert(!trackOop->hasPromotedMark(), "Should not have been marked");
1.153 + trackOop->setPromotedMark();
1.154 +}
1.155 +
1.156 +// Save the given displaced header, incrementing the pointer and
1.157 +// obtaining more spool area as necessary.
1.158 +void PromotionInfo::saveDisplacedHeader(markOop hdr) {
1.159 + assert(_spoolHead != NULL && _spoolTail != NULL,
1.160 + "promotionInfo inconsistency");
1.161 + assert(_spoolTail->bufferSize > _nextIndex, "Off by one error at tail?");
1.162 + _spoolTail->displacedHdr[_nextIndex] = hdr;
1.163 + // Spool forward
1.164 + if (++_nextIndex == _spoolTail->bufferSize) { // last location in this block
1.165 + // get a new spooling block
1.166 + assert(_spoolTail->nextSpoolBlock == NULL, "tail should terminate spool list");
1.167 + _splice_point = _spoolTail; // save for splicing
1.168 + _spoolTail->nextSpoolBlock = getSpoolBlock(); // might fail
1.169 + _spoolTail = _spoolTail->nextSpoolBlock; // might become NULL ...
1.170 + // ... but will attempt filling before next promotion attempt
1.171 + _nextIndex = 1;
1.172 + }
1.173 +}
1.174 +
1.175 +// Ensure that spooling space exists. Return false if spooling space
1.176 +// could not be obtained.
1.177 +bool PromotionInfo::ensure_spooling_space_work() {
1.178 + assert(!has_spooling_space(), "Only call when there is no spooling space");
1.179 + // Try and obtain more spooling space
1.180 + SpoolBlock* newSpool = getSpoolBlock();
1.181 + assert(newSpool == NULL ||
1.182 + (newSpool->bufferSize != 0 && newSpool->nextSpoolBlock == NULL),
1.183 + "getSpoolBlock() sanity check");
1.184 + if (newSpool == NULL) {
1.185 + return false;
1.186 + }
1.187 + _nextIndex = 1;
1.188 + if (_spoolTail == NULL) {
1.189 + _spoolTail = newSpool;
1.190 + if (_spoolHead == NULL) {
1.191 + _spoolHead = newSpool;
1.192 + _firstIndex = 1;
1.193 + } else {
1.194 + assert(_splice_point != NULL && _splice_point->nextSpoolBlock == NULL,
1.195 + "Splice point invariant");
1.196 + // Extra check that _splice_point is connected to list
1.197 + #ifdef ASSERT
1.198 + {
1.199 + SpoolBlock* blk = _spoolHead;
1.200 + for (; blk->nextSpoolBlock != NULL;
1.201 + blk = blk->nextSpoolBlock);
1.202 + assert(blk != NULL && blk == _splice_point,
1.203 + "Splice point incorrect");
1.204 + }
1.205 + #endif // ASSERT
1.206 + _splice_point->nextSpoolBlock = newSpool;
1.207 + }
1.208 + } else {
1.209 + assert(_spoolHead != NULL, "spool list consistency");
1.210 + _spoolTail->nextSpoolBlock = newSpool;
1.211 + _spoolTail = newSpool;
1.212 + }
1.213 + return true;
1.214 +}
1.215 +
1.216 +// Get a free spool buffer from the free pool, getting a new block
1.217 +// from the heap if necessary.
1.218 +SpoolBlock* PromotionInfo::getSpoolBlock() {
1.219 + SpoolBlock* res;
1.220 + if ((res = _spareSpool) != NULL) {
1.221 + _spareSpool = _spareSpool->nextSpoolBlock;
1.222 + res->nextSpoolBlock = NULL;
1.223 + } else { // spare spool exhausted, get some from heap
1.224 + res = (SpoolBlock*)(space()->allocateScratch(refillSize()));
1.225 + if (res != NULL) {
1.226 + res->init();
1.227 + }
1.228 + }
1.229 + assert(res == NULL || res->nextSpoolBlock == NULL, "postcondition");
1.230 + return res;
1.231 +}
1.232 +
1.233 +void PromotionInfo::startTrackingPromotions() {
1.234 + assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex,
1.235 + "spooling inconsistency?");
1.236 + _firstIndex = _nextIndex = 1;
1.237 + _tracking = true;
1.238 +}
1.239 +
1.240 +#define CMSPrintPromoBlockInfo 1
1.241 +
1.242 +void PromotionInfo::stopTrackingPromotions(uint worker_id) {
1.243 + assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex,
1.244 + "spooling inconsistency?");
1.245 + _firstIndex = _nextIndex = 1;
1.246 + _tracking = false;
1.247 + if (CMSPrintPromoBlockInfo > 1) {
1.248 + print_statistics(worker_id);
1.249 + }
1.250 +}
1.251 +
1.252 +void PromotionInfo::print_statistics(uint worker_id) const {
1.253 + assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex,
1.254 + "Else will undercount");
1.255 + assert(CMSPrintPromoBlockInfo > 0, "Else unnecessary call");
1.256 + // Count the number of blocks and slots in the free pool
1.257 + size_t slots = 0;
1.258 + size_t blocks = 0;
1.259 + for (SpoolBlock* cur_spool = _spareSpool;
1.260 + cur_spool != NULL;
1.261 + cur_spool = cur_spool->nextSpoolBlock) {
1.262 + // the first entry is just a self-pointer; indices 1 through
1.263 + // bufferSize - 1 are occupied (thus, bufferSize - 1 slots).
1.264 + assert((void*)cur_spool->displacedHdr == (void*)&cur_spool->displacedHdr,
1.265 + "first entry of displacedHdr should be self-referential");
1.266 + slots += cur_spool->bufferSize - 1;
1.267 + blocks++;
1.268 + }
1.269 + if (_spoolHead != NULL) {
1.270 + slots += _spoolHead->bufferSize - 1;
1.271 + blocks++;
1.272 + }
1.273 + gclog_or_tty->print_cr(" [worker %d] promo_blocks = %d, promo_slots = %d ",
1.274 + worker_id, blocks, slots);
1.275 +}
1.276 +
1.277 +// When _spoolTail is not NULL, then the slot <_spoolTail, _nextIndex>
1.278 +// points to the next slot available for filling.
1.279 +// The set of slots holding displaced headers are then all those in the
1.280 +// right-open interval denoted by:
1.281 +//
1.282 +// [ <_spoolHead, _firstIndex>, <_spoolTail, _nextIndex> )
1.283 +//
1.284 +// When _spoolTail is NULL, then the set of slots with displaced headers
1.285 +// is all those starting at the slot <_spoolHead, _firstIndex> and
1.286 +// going up to the last slot of last block in the linked list.
1.287 +// In this lartter case, _splice_point points to the tail block of
1.288 +// this linked list of blocks holding displaced headers.
1.289 +void PromotionInfo::verify() const {
1.290 + // Verify the following:
1.291 + // 1. the number of displaced headers matches the number of promoted
1.292 + // objects that have displaced headers
1.293 + // 2. each promoted object lies in this space
1.294 + debug_only(
1.295 + PromotedObject* junk = NULL;
1.296 + assert(junk->next_addr() == (void*)(oop(junk)->mark_addr()),
1.297 + "Offset of PromotedObject::_next is expected to align with "
1.298 + " the OopDesc::_mark within OopDesc");
1.299 + )
1.300 + // FIXME: guarantee????
1.301 + guarantee(_spoolHead == NULL || _spoolTail != NULL ||
1.302 + _splice_point != NULL, "list consistency");
1.303 + guarantee(_promoHead == NULL || _promoTail != NULL, "list consistency");
1.304 + // count the number of objects with displaced headers
1.305 + size_t numObjsWithDisplacedHdrs = 0;
1.306 + for (PromotedObject* curObj = _promoHead; curObj != NULL; curObj = curObj->next()) {
1.307 + guarantee(space()->is_in_reserved((HeapWord*)curObj), "Containment");
1.308 + // the last promoted object may fail the mark() != NULL test of is_oop().
1.309 + guarantee(curObj->next() == NULL || oop(curObj)->is_oop(), "must be an oop");
1.310 + if (curObj->hasDisplacedMark()) {
1.311 + numObjsWithDisplacedHdrs++;
1.312 + }
1.313 + }
1.314 + // Count the number of displaced headers
1.315 + size_t numDisplacedHdrs = 0;
1.316 + for (SpoolBlock* curSpool = _spoolHead;
1.317 + curSpool != _spoolTail && curSpool != NULL;
1.318 + curSpool = curSpool->nextSpoolBlock) {
1.319 + // the first entry is just a self-pointer; indices 1 through
1.320 + // bufferSize - 1 are occupied (thus, bufferSize - 1 slots).
1.321 + guarantee((void*)curSpool->displacedHdr == (void*)&curSpool->displacedHdr,
1.322 + "first entry of displacedHdr should be self-referential");
1.323 + numDisplacedHdrs += curSpool->bufferSize - 1;
1.324 + }
1.325 + guarantee((_spoolHead == _spoolTail) == (numDisplacedHdrs == 0),
1.326 + "internal consistency");
1.327 + guarantee(_spoolTail != NULL || _nextIndex == 1,
1.328 + "Inconsistency between _spoolTail and _nextIndex");
1.329 + // We overcounted (_firstIndex-1) worth of slots in block
1.330 + // _spoolHead and we undercounted (_nextIndex-1) worth of
1.331 + // slots in block _spoolTail. We make an appropriate
1.332 + // adjustment by subtracting the first and adding the
1.333 + // second: - (_firstIndex - 1) + (_nextIndex - 1)
1.334 + numDisplacedHdrs += (_nextIndex - _firstIndex);
1.335 + guarantee(numDisplacedHdrs == numObjsWithDisplacedHdrs, "Displaced hdr count");
1.336 +}
1.337 +
1.338 +void PromotionInfo::print_on(outputStream* st) const {
1.339 + SpoolBlock* curSpool = NULL;
1.340 + size_t i = 0;
1.341 + st->print_cr(" start & end indices: [" SIZE_FORMAT ", " SIZE_FORMAT ")",
1.342 + _firstIndex, _nextIndex);
1.343 + for (curSpool = _spoolHead; curSpool != _spoolTail && curSpool != NULL;
1.344 + curSpool = curSpool->nextSpoolBlock) {
1.345 + curSpool->print_on(st);
1.346 + st->print_cr(" active ");
1.347 + i++;
1.348 + }
1.349 + for (curSpool = _spoolTail; curSpool != NULL;
1.350 + curSpool = curSpool->nextSpoolBlock) {
1.351 + curSpool->print_on(st);
1.352 + st->print_cr(" inactive ");
1.353 + i++;
1.354 + }
1.355 + for (curSpool = _spareSpool; curSpool != NULL;
1.356 + curSpool = curSpool->nextSpoolBlock) {
1.357 + curSpool->print_on(st);
1.358 + st->print_cr(" free ");
1.359 + i++;
1.360 + }
1.361 + st->print_cr(" " SIZE_FORMAT " header spooling blocks", i);
1.362 +}
1.363 +
1.364 +void SpoolBlock::print_on(outputStream* st) const {
1.365 + st->print("[" PTR_FORMAT "," PTR_FORMAT "), " SIZE_FORMAT " HeapWords -> " PTR_FORMAT,
1.366 + this, (HeapWord*)displacedHdr + bufferSize,
1.367 + bufferSize, nextSpoolBlock);
1.368 +}
