Mon, 06 Aug 2012 12:20:14 -0700
6818524: G1: use ergonomic resizing of PLABs
Summary: Employ PLABStats instances to record information about survivor and old PLABs, and use the recorded stats to adjust the sizes of survivor and old PLABS.
Reviewed-by: johnc, ysr
Contributed-by: Brandon Mitchell <brandon@twitter.com>
1.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Fri Aug 03 13:24:02 2012 -0700 1.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Mon Aug 06 12:20:14 2012 -0700 1.3 @@ -1891,6 +1891,8 @@ 1.4 _young_list(new YoungList(this)), 1.5 _gc_time_stamp(0), 1.6 _retained_old_gc_alloc_region(NULL), 1.7 + _survivor_plab_stats(YoungPLABSize, PLABWeight), 1.8 + _old_plab_stats(OldPLABSize, PLABWeight), 1.9 _expand_heap_after_alloc_failure(true), 1.10 _surviving_young_words(NULL), 1.11 _old_marking_cycles_started(0), 1.12 @@ -4099,17 +4101,22 @@ 1.13 size_t gclab_word_size; 1.14 switch (purpose) { 1.15 case GCAllocForSurvived: 1.16 - gclab_word_size = YoungPLABSize; 1.17 + gclab_word_size = _survivor_plab_stats.desired_plab_sz(); 1.18 break; 1.19 case GCAllocForTenured: 1.20 - gclab_word_size = OldPLABSize; 1.21 + gclab_word_size = _old_plab_stats.desired_plab_sz(); 1.22 break; 1.23 default: 1.24 assert(false, "unknown GCAllocPurpose"); 1.25 - gclab_word_size = OldPLABSize; 1.26 + gclab_word_size = _old_plab_stats.desired_plab_sz(); 1.27 break; 1.28 } 1.29 - return gclab_word_size; 1.30 + 1.31 + // Prevent humongous PLAB sizes for two reasons: 1.32 + // * PLABs are allocated using a similar paths as oops, but should 1.33 + // never be in a humongous region 1.34 + // * Allowing humongous PLABs needlessly churns the region free lists 1.35 + return MIN2(_humongous_object_threshold_in_words, gclab_word_size); 1.36 } 1.37 1.38 void G1CollectedHeap::init_mutator_alloc_region() { 1.39 @@ -4165,6 +4172,11 @@ 1.40 // want either way so no reason to check explicitly for either 1.41 // condition. 1.42 _retained_old_gc_alloc_region = _old_gc_alloc_region.release(); 1.43 + 1.44 + if (ResizePLAB) { 1.45 + _survivor_plab_stats.adjust_desired_plab_sz(); 1.46 + _old_plab_stats.adjust_desired_plab_sz(); 1.47 + } 1.48 } 1.49 1.50 void G1CollectedHeap::abandon_gc_alloc_regions() {
2.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Fri Aug 03 13:24:02 2012 -0700 2.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Mon Aug 06 12:20:14 2012 -0700 2.3 @@ -33,7 +33,7 @@ 2.4 #include "gc_implementation/g1/heapRegionSeq.hpp" 2.5 #include "gc_implementation/g1/heapRegionSets.hpp" 2.6 #include "gc_implementation/shared/hSpaceCounters.hpp" 2.7 -#include "gc_implementation/parNew/parGCAllocBuffer.hpp" 2.8 +#include "gc_implementation/shared/parGCAllocBuffer.hpp" 2.9 #include "memory/barrierSet.hpp" 2.10 #include "memory/memRegion.hpp" 2.11 #include "memory/sharedHeap.hpp" 2.12 @@ -278,10 +278,33 @@ 2.13 // survivor objects. 2.14 SurvivorGCAllocRegion _survivor_gc_alloc_region; 2.15 2.16 + // PLAB sizing policy for survivors. 2.17 + PLABStats _survivor_plab_stats; 2.18 + 2.19 // Alloc region used to satisfy allocation requests by the GC for 2.20 // old objects. 2.21 OldGCAllocRegion _old_gc_alloc_region; 2.22 2.23 + // PLAB sizing policy for tenured objects. 2.24 + PLABStats _old_plab_stats; 2.25 + 2.26 + PLABStats* stats_for_purpose(GCAllocPurpose purpose) { 2.27 + PLABStats* stats = NULL; 2.28 + 2.29 + switch (purpose) { 2.30 + case GCAllocForSurvived: 2.31 + stats = &_survivor_plab_stats; 2.32 + break; 2.33 + case GCAllocForTenured: 2.34 + stats = &_old_plab_stats; 2.35 + break; 2.36 + default: 2.37 + assert(false, "unrecognized GCAllocPurpose"); 2.38 + } 2.39 + 2.40 + return stats; 2.41 + } 2.42 + 2.43 // The last old region we allocated to during the last GC. 2.44 // Typically, it is not full so we should re-use it during the next GC. 2.45 HeapRegion* _retained_old_gc_alloc_region; 2.46 @@ -314,7 +337,7 @@ 2.47 G1MonitoringSupport* _g1mm; 2.48 2.49 // Determines PLAB size for a particular allocation purpose. 2.50 - static size_t desired_plab_sz(GCAllocPurpose purpose); 2.51 + size_t desired_plab_sz(GCAllocPurpose purpose); 2.52 2.53 // Outside of GC pauses, the number of bytes used in all regions other 2.54 // than the current allocation region. 2.55 @@ -1811,19 +1834,19 @@ 2.56 } 2.57 2.58 HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) { 2.59 - 2.60 HeapWord* obj = NULL; 2.61 size_t gclab_word_size = _g1h->desired_plab_sz(purpose); 2.62 if (word_sz * 100 < gclab_word_size * ParallelGCBufferWastePct) { 2.63 G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose); 2.64 - assert(gclab_word_size == alloc_buf->word_sz(), 2.65 - "dynamic resizing is not supported"); 2.66 add_to_alloc_buffer_waste(alloc_buf->words_remaining()); 2.67 - alloc_buf->retire(false, false); 2.68 + alloc_buf->flush_stats_and_retire(_g1h->stats_for_purpose(purpose), 2.69 + false /* end_of_gc */, 2.70 + false /* retain */); 2.71 2.72 HeapWord* buf = _g1h->par_allocate_during_gc(purpose, gclab_word_size); 2.73 if (buf == NULL) return NULL; // Let caller handle allocation failure. 2.74 // Otherwise. 2.75 + alloc_buf->set_word_size(gclab_word_size); 2.76 alloc_buf->set_buf(buf); 2.77 2.78 obj = alloc_buf->allocate(word_sz); 2.79 @@ -1908,7 +1931,9 @@ 2.80 for (int ap = 0; ap < GCAllocPurposeCount; ++ap) { 2.81 size_t waste = _alloc_buffers[ap]->words_remaining(); 2.82 add_to_alloc_buffer_waste(waste); 2.83 - _alloc_buffers[ap]->retire(true, false); 2.84 + _alloc_buffers[ap]->flush_stats_and_retire(_g1h->stats_for_purpose((GCAllocPurpose)ap), 2.85 + true /* end_of_gc */, 2.86 + false /* retain */); 2.87 } 2.88 } 2.89
3.1 --- a/src/share/vm/gc_implementation/parNew/parGCAllocBuffer.cpp Fri Aug 03 13:24:02 2012 -0700 3.2 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 3.3 @@ -1,344 +0,0 @@ 3.4 -/* 3.5 - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 3.6 - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3.7 - * 3.8 - * This code is free software; you can redistribute it and/or modify it 3.9 - * under the terms of the GNU General Public License version 2 only, as 3.10 - * published by the Free Software Foundation. 3.11 - * 3.12 - * This code is distributed in the hope that it will be useful, but WITHOUT 3.13 - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 3.14 - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 3.15 - * version 2 for more details (a copy is included in the LICENSE file that 3.16 - * accompanied this code). 3.17 - * 3.18 - * You should have received a copy of the GNU General Public License version 3.19 - * 2 along with this work; if not, write to the Free Software Foundation, 3.20 - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 3.21 - * 3.22 - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 3.23 - * or visit www.oracle.com if you need additional information or have any 3.24 - * questions. 3.25 - * 3.26 - */ 3.27 - 3.28 -#include "precompiled.hpp" 3.29 -#include "gc_implementation/parNew/parGCAllocBuffer.hpp" 3.30 -#include "memory/sharedHeap.hpp" 3.31 -#include "oops/arrayOop.hpp" 3.32 -#include "oops/oop.inline.hpp" 3.33 - 3.34 -ParGCAllocBuffer::ParGCAllocBuffer(size_t desired_plab_sz_) : 3.35 - _word_sz(desired_plab_sz_), _bottom(NULL), _top(NULL), 3.36 - _end(NULL), _hard_end(NULL), 3.37 - _retained(false), _retained_filler(), 3.38 - _allocated(0), _wasted(0) 3.39 -{ 3.40 - assert (min_size() > AlignmentReserve, "Inconsistency!"); 3.41 - // arrayOopDesc::header_size depends on command line initialization. 3.42 - FillerHeaderSize = align_object_size(arrayOopDesc::header_size(T_INT)); 3.43 - AlignmentReserve = oopDesc::header_size() > MinObjAlignment ? FillerHeaderSize : 0; 3.44 -} 3.45 - 3.46 -size_t ParGCAllocBuffer::FillerHeaderSize; 3.47 - 3.48 -// If the minimum object size is greater than MinObjAlignment, we can 3.49 -// end up with a shard at the end of the buffer that's smaller than 3.50 -// the smallest object. We can't allow that because the buffer must 3.51 -// look like it's full of objects when we retire it, so we make 3.52 -// sure we have enough space for a filler int array object. 3.53 -size_t ParGCAllocBuffer::AlignmentReserve; 3.54 - 3.55 -void ParGCAllocBuffer::retire(bool end_of_gc, bool retain) { 3.56 - assert(!retain || end_of_gc, "Can only retain at GC end."); 3.57 - if (_retained) { 3.58 - // If the buffer had been retained shorten the previous filler object. 3.59 - assert(_retained_filler.end() <= _top, "INVARIANT"); 3.60 - CollectedHeap::fill_with_object(_retained_filler); 3.61 - // Wasted space book-keeping, otherwise (normally) done in invalidate() 3.62 - _wasted += _retained_filler.word_size(); 3.63 - _retained = false; 3.64 - } 3.65 - assert(!end_of_gc || !_retained, "At this point, end_of_gc ==> !_retained."); 3.66 - if (_top < _hard_end) { 3.67 - CollectedHeap::fill_with_object(_top, _hard_end); 3.68 - if (!retain) { 3.69 - invalidate(); 3.70 - } else { 3.71 - // Is there wasted space we'd like to retain for the next GC? 3.72 - if (pointer_delta(_end, _top) > FillerHeaderSize) { 3.73 - _retained = true; 3.74 - _retained_filler = MemRegion(_top, FillerHeaderSize); 3.75 - _top = _top + FillerHeaderSize; 3.76 - } else { 3.77 - invalidate(); 3.78 - } 3.79 - } 3.80 - } 3.81 -} 3.82 - 3.83 -void ParGCAllocBuffer::flush_stats(PLABStats* stats) { 3.84 - assert(ResizePLAB, "Wasted work"); 3.85 - stats->add_allocated(_allocated); 3.86 - stats->add_wasted(_wasted); 3.87 - stats->add_unused(pointer_delta(_end, _top)); 3.88 -} 3.89 - 3.90 -// Compute desired plab size and latch result for later 3.91 -// use. This should be called once at the end of parallel 3.92 -// scavenge; it clears the sensor accumulators. 3.93 -void PLABStats::adjust_desired_plab_sz() { 3.94 - assert(ResizePLAB, "Not set"); 3.95 - if (_allocated == 0) { 3.96 - assert(_unused == 0, "Inconsistency in PLAB stats"); 3.97 - _allocated = 1; 3.98 - } 3.99 - double wasted_frac = (double)_unused/(double)_allocated; 3.100 - size_t target_refills = (size_t)((wasted_frac*TargetSurvivorRatio)/ 3.101 - TargetPLABWastePct); 3.102 - if (target_refills == 0) { 3.103 - target_refills = 1; 3.104 - } 3.105 - _used = _allocated - _wasted - _unused; 3.106 - size_t plab_sz = _used/(target_refills*ParallelGCThreads); 3.107 - if (PrintPLAB) gclog_or_tty->print(" (plab_sz = %d ", plab_sz); 3.108 - // Take historical weighted average 3.109 - _filter.sample(plab_sz); 3.110 - // Clip from above and below, and align to object boundary 3.111 - plab_sz = MAX2(min_size(), (size_t)_filter.average()); 3.112 - plab_sz = MIN2(max_size(), plab_sz); 3.113 - plab_sz = align_object_size(plab_sz); 3.114 - // Latch the result 3.115 - if (PrintPLAB) gclog_or_tty->print(" desired_plab_sz = %d) ", plab_sz); 3.116 - if (ResizePLAB) { 3.117 - _desired_plab_sz = plab_sz; 3.118 - } 3.119 - // Now clear the accumulators for next round: 3.120 - // note this needs to be fixed in the case where we 3.121 - // are retaining across scavenges. FIX ME !!! XXX 3.122 - _allocated = 0; 3.123 - _wasted = 0; 3.124 - _unused = 0; 3.125 -} 3.126 - 3.127 -#ifndef PRODUCT 3.128 -void ParGCAllocBuffer::print() { 3.129 - gclog_or_tty->print("parGCAllocBuffer: _bottom: %p _top: %p _end: %p _hard_end: %p" 3.130 - "_retained: %c _retained_filler: [%p,%p)\n", 3.131 - _bottom, _top, _end, _hard_end, 3.132 - "FT"[_retained], _retained_filler.start(), _retained_filler.end()); 3.133 -} 3.134 -#endif // !PRODUCT 3.135 - 3.136 -const size_t ParGCAllocBufferWithBOT::ChunkSizeInWords = 3.137 -MIN2(CardTableModRefBS::par_chunk_heapword_alignment(), 3.138 - ((size_t)Generation::GenGrain)/HeapWordSize); 3.139 -const size_t ParGCAllocBufferWithBOT::ChunkSizeInBytes = 3.140 -MIN2(CardTableModRefBS::par_chunk_heapword_alignment() * HeapWordSize, 3.141 - (size_t)Generation::GenGrain); 3.142 - 3.143 -ParGCAllocBufferWithBOT::ParGCAllocBufferWithBOT(size_t word_sz, 3.144 - BlockOffsetSharedArray* bsa) : 3.145 - ParGCAllocBuffer(word_sz), 3.146 - _bsa(bsa), 3.147 - _bt(bsa, MemRegion(_bottom, _hard_end)), 3.148 - _true_end(_hard_end) 3.149 -{} 3.150 - 3.151 -// The buffer comes with its own BOT, with a shared (obviously) underlying 3.152 -// BlockOffsetSharedArray. We manipulate this BOT in the normal way 3.153 -// as we would for any contiguous space. However, on accasion we 3.154 -// need to do some buffer surgery at the extremities before we 3.155 -// start using the body of the buffer for allocations. Such surgery 3.156 -// (as explained elsewhere) is to prevent allocation on a card that 3.157 -// is in the process of being walked concurrently by another GC thread. 3.158 -// When such surgery happens at a point that is far removed (to the 3.159 -// right of the current allocation point, top), we use the "contig" 3.160 -// parameter below to directly manipulate the shared array without 3.161 -// modifying the _next_threshold state in the BOT. 3.162 -void ParGCAllocBufferWithBOT::fill_region_with_block(MemRegion mr, 3.163 - bool contig) { 3.164 - CollectedHeap::fill_with_object(mr); 3.165 - if (contig) { 3.166 - _bt.alloc_block(mr.start(), mr.end()); 3.167 - } else { 3.168 - _bt.BlockOffsetArray::alloc_block(mr.start(), mr.end()); 3.169 - } 3.170 -} 3.171 - 3.172 -HeapWord* ParGCAllocBufferWithBOT::allocate_slow(size_t word_sz) { 3.173 - HeapWord* res = NULL; 3.174 - if (_true_end > _hard_end) { 3.175 - assert((HeapWord*)align_size_down(intptr_t(_hard_end), 3.176 - ChunkSizeInBytes) == _hard_end, 3.177 - "or else _true_end should be equal to _hard_end"); 3.178 - assert(_retained, "or else _true_end should be equal to _hard_end"); 3.179 - assert(_retained_filler.end() <= _top, "INVARIANT"); 3.180 - CollectedHeap::fill_with_object(_retained_filler); 3.181 - if (_top < _hard_end) { 3.182 - fill_region_with_block(MemRegion(_top, _hard_end), true); 3.183 - } 3.184 - HeapWord* next_hard_end = MIN2(_true_end, _hard_end + ChunkSizeInWords); 3.185 - _retained_filler = MemRegion(_hard_end, FillerHeaderSize); 3.186 - _bt.alloc_block(_retained_filler.start(), _retained_filler.word_size()); 3.187 - _top = _retained_filler.end(); 3.188 - _hard_end = next_hard_end; 3.189 - _end = _hard_end - AlignmentReserve; 3.190 - res = ParGCAllocBuffer::allocate(word_sz); 3.191 - if (res != NULL) { 3.192 - _bt.alloc_block(res, word_sz); 3.193 - } 3.194 - } 3.195 - return res; 3.196 -} 3.197 - 3.198 -void 3.199 -ParGCAllocBufferWithBOT::undo_allocation(HeapWord* obj, size_t word_sz) { 3.200 - ParGCAllocBuffer::undo_allocation(obj, word_sz); 3.201 - // This may back us up beyond the previous threshold, so reset. 3.202 - _bt.set_region(MemRegion(_top, _hard_end)); 3.203 - _bt.initialize_threshold(); 3.204 -} 3.205 - 3.206 -void ParGCAllocBufferWithBOT::retire(bool end_of_gc, bool retain) { 3.207 - assert(!retain || end_of_gc, "Can only retain at GC end."); 3.208 - if (_retained) { 3.209 - // We're about to make the retained_filler into a block. 3.210 - _bt.BlockOffsetArray::alloc_block(_retained_filler.start(), 3.211 - _retained_filler.end()); 3.212 - } 3.213 - // Reset _hard_end to _true_end (and update _end) 3.214 - if (retain && _hard_end != NULL) { 3.215 - assert(_hard_end <= _true_end, "Invariant."); 3.216 - _hard_end = _true_end; 3.217 - _end = MAX2(_top, _hard_end - AlignmentReserve); 3.218 - assert(_end <= _hard_end, "Invariant."); 3.219 - } 3.220 - _true_end = _hard_end; 3.221 - HeapWord* pre_top = _top; 3.222 - 3.223 - ParGCAllocBuffer::retire(end_of_gc, retain); 3.224 - // Now any old _retained_filler is cut back to size, the free part is 3.225 - // filled with a filler object, and top is past the header of that 3.226 - // object. 3.227 - 3.228 - if (retain && _top < _end) { 3.229 - assert(end_of_gc && retain, "Or else retain should be false."); 3.230 - // If the lab does not start on a card boundary, we don't want to 3.231 - // allocate onto that card, since that might lead to concurrent 3.232 - // allocation and card scanning, which we don't support. So we fill 3.233 - // the first card with a garbage object. 3.234 - size_t first_card_index = _bsa->index_for(pre_top); 3.235 - HeapWord* first_card_start = _bsa->address_for_index(first_card_index); 3.236 - if (first_card_start < pre_top) { 3.237 - HeapWord* second_card_start = 3.238 - _bsa->inc_by_region_size(first_card_start); 3.239 - 3.240 - // Ensure enough room to fill with the smallest block 3.241 - second_card_start = MAX2(second_card_start, pre_top + AlignmentReserve); 3.242 - 3.243 - // If the end is already in the first card, don't go beyond it! 3.244 - // Or if the remainder is too small for a filler object, gobble it up. 3.245 - if (_hard_end < second_card_start || 3.246 - pointer_delta(_hard_end, second_card_start) < AlignmentReserve) { 3.247 - second_card_start = _hard_end; 3.248 - } 3.249 - if (pre_top < second_card_start) { 3.250 - MemRegion first_card_suffix(pre_top, second_card_start); 3.251 - fill_region_with_block(first_card_suffix, true); 3.252 - } 3.253 - pre_top = second_card_start; 3.254 - _top = pre_top; 3.255 - _end = MAX2(_top, _hard_end - AlignmentReserve); 3.256 - } 3.257 - 3.258 - // If the lab does not end on a card boundary, we don't want to 3.259 - // allocate onto that card, since that might lead to concurrent 3.260 - // allocation and card scanning, which we don't support. So we fill 3.261 - // the last card with a garbage object. 3.262 - size_t last_card_index = _bsa->index_for(_hard_end); 3.263 - HeapWord* last_card_start = _bsa->address_for_index(last_card_index); 3.264 - if (last_card_start < _hard_end) { 3.265 - 3.266 - // Ensure enough room to fill with the smallest block 3.267 - last_card_start = MIN2(last_card_start, _hard_end - AlignmentReserve); 3.268 - 3.269 - // If the top is already in the last card, don't go back beyond it! 3.270 - // Or if the remainder is too small for a filler object, gobble it up. 3.271 - if (_top > last_card_start || 3.272 - pointer_delta(last_card_start, _top) < AlignmentReserve) { 3.273 - last_card_start = _top; 3.274 - } 3.275 - if (last_card_start < _hard_end) { 3.276 - MemRegion last_card_prefix(last_card_start, _hard_end); 3.277 - fill_region_with_block(last_card_prefix, false); 3.278 - } 3.279 - _hard_end = last_card_start; 3.280 - _end = MAX2(_top, _hard_end - AlignmentReserve); 3.281 - _true_end = _hard_end; 3.282 - assert(_end <= _hard_end, "Invariant."); 3.283 - } 3.284 - 3.285 - // At this point: 3.286 - // 1) we had a filler object from the original top to hard_end. 3.287 - // 2) We've filled in any partial cards at the front and back. 3.288 - if (pre_top < _hard_end) { 3.289 - // Now we can reset the _bt to do allocation in the given area. 3.290 - MemRegion new_filler(pre_top, _hard_end); 3.291 - fill_region_with_block(new_filler, false); 3.292 - _top = pre_top + ParGCAllocBuffer::FillerHeaderSize; 3.293 - // If there's no space left, don't retain. 3.294 - if (_top >= _end) { 3.295 - _retained = false; 3.296 - invalidate(); 3.297 - return; 3.298 - } 3.299 - _retained_filler = MemRegion(pre_top, _top); 3.300 - _bt.set_region(MemRegion(_top, _hard_end)); 3.301 - _bt.initialize_threshold(); 3.302 - assert(_bt.threshold() > _top, "initialize_threshold failed!"); 3.303 - 3.304 - // There may be other reasons for queries into the middle of the 3.305 - // filler object. When such queries are done in parallel with 3.306 - // allocation, bad things can happen, if the query involves object 3.307 - // iteration. So we ensure that such queries do not involve object 3.308 - // iteration, by putting another filler object on the boundaries of 3.309 - // such queries. One such is the object spanning a parallel card 3.310 - // chunk boundary. 3.311 - 3.312 - // "chunk_boundary" is the address of the first chunk boundary less 3.313 - // than "hard_end". 3.314 - HeapWord* chunk_boundary = 3.315 - (HeapWord*)align_size_down(intptr_t(_hard_end-1), ChunkSizeInBytes); 3.316 - assert(chunk_boundary < _hard_end, "Or else above did not work."); 3.317 - assert(pointer_delta(_true_end, chunk_boundary) >= AlignmentReserve, 3.318 - "Consequence of last card handling above."); 3.319 - 3.320 - if (_top <= chunk_boundary) { 3.321 - assert(_true_end == _hard_end, "Invariant."); 3.322 - while (_top <= chunk_boundary) { 3.323 - assert(pointer_delta(_hard_end, chunk_boundary) >= AlignmentReserve, 3.324 - "Consequence of last card handling above."); 3.325 - _bt.BlockOffsetArray::alloc_block(chunk_boundary, _hard_end); 3.326 - CollectedHeap::fill_with_object(chunk_boundary, _hard_end); 3.327 - _hard_end = chunk_boundary; 3.328 - chunk_boundary -= ChunkSizeInWords; 3.329 - } 3.330 - _end = _hard_end - AlignmentReserve; 3.331 - assert(_top <= _end, "Invariant."); 3.332 - // Now reset the initial filler chunk so it doesn't overlap with 3.333 - // the one(s) inserted above. 3.334 - MemRegion new_filler(pre_top, _hard_end); 3.335 - fill_region_with_block(new_filler, false); 3.336 - } 3.337 - } else { 3.338 - _retained = false; 3.339 - invalidate(); 3.340 - } 3.341 - } else { 3.342 - assert(!end_of_gc || 3.343 - (!_retained && _true_end == _hard_end), "Checking."); 3.344 - } 3.345 - assert(_end <= _hard_end, "Invariant."); 3.346 - assert(_top < _end || _top == _hard_end, "Invariant"); 3.347 -}
4.1 --- a/src/share/vm/gc_implementation/parNew/parGCAllocBuffer.hpp Fri Aug 03 13:24:02 2012 -0700 4.2 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 4.3 @@ -1,249 +0,0 @@ 4.4 -/* 4.5 - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 4.6 - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4.7 - * 4.8 - * This code is free software; you can redistribute it and/or modify it 4.9 - * under the terms of the GNU General Public License version 2 only, as 4.10 - * published by the Free Software Foundation. 4.11 - * 4.12 - * This code is distributed in the hope that it will be useful, but WITHOUT 4.13 - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 4.14 - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 4.15 - * version 2 for more details (a copy is included in the LICENSE file that 4.16 - * accompanied this code). 4.17 - * 4.18 - * You should have received a copy of the GNU General Public License version 4.19 - * 2 along with this work; if not, write to the Free Software Foundation, 4.20 - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 4.21 - * 4.22 - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 4.23 - * or visit www.oracle.com if you need additional information or have any 4.24 - * questions. 4.25 - * 4.26 - */ 4.27 - 4.28 -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP 4.29 -#define SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP 4.30 - 4.31 -#include "memory/allocation.hpp" 4.32 -#include "memory/blockOffsetTable.hpp" 4.33 -#include "memory/threadLocalAllocBuffer.hpp" 4.34 -#include "utilities/globalDefinitions.hpp" 4.35 - 4.36 -// Forward decl. 4.37 - 4.38 -class PLABStats; 4.39 - 4.40 -// A per-thread allocation buffer used during GC. 4.41 -class ParGCAllocBuffer: public CHeapObj<mtGC> { 4.42 -protected: 4.43 - char head[32]; 4.44 - size_t _word_sz; // in HeapWord units 4.45 - HeapWord* _bottom; 4.46 - HeapWord* _top; 4.47 - HeapWord* _end; // last allocatable address + 1 4.48 - HeapWord* _hard_end; // _end + AlignmentReserve 4.49 - bool _retained; // whether we hold a _retained_filler 4.50 - MemRegion _retained_filler; 4.51 - // In support of ergonomic sizing of PLAB's 4.52 - size_t _allocated; // in HeapWord units 4.53 - size_t _wasted; // in HeapWord units 4.54 - char tail[32]; 4.55 - static size_t FillerHeaderSize; 4.56 - static size_t AlignmentReserve; 4.57 - 4.58 -public: 4.59 - // Initializes the buffer to be empty, but with the given "word_sz". 4.60 - // Must get initialized with "set_buf" for an allocation to succeed. 4.61 - ParGCAllocBuffer(size_t word_sz); 4.62 - 4.63 - static const size_t min_size() { 4.64 - return ThreadLocalAllocBuffer::min_size(); 4.65 - } 4.66 - 4.67 - static const size_t max_size() { 4.68 - return ThreadLocalAllocBuffer::max_size(); 4.69 - } 4.70 - 4.71 - // If an allocation of the given "word_sz" can be satisfied within the 4.72 - // buffer, do the allocation, returning a pointer to the start of the 4.73 - // allocated block. If the allocation request cannot be satisfied, 4.74 - // return NULL. 4.75 - HeapWord* allocate(size_t word_sz) { 4.76 - HeapWord* res = _top; 4.77 - if (pointer_delta(_end, _top) >= word_sz) { 4.78 - _top = _top + word_sz; 4.79 - return res; 4.80 - } else { 4.81 - return NULL; 4.82 - } 4.83 - } 4.84 - 4.85 - // Undo the last allocation in the buffer, which is required to be of the 4.86 - // "obj" of the given "word_sz". 4.87 - void undo_allocation(HeapWord* obj, size_t word_sz) { 4.88 - assert(pointer_delta(_top, _bottom) >= word_sz, "Bad undo"); 4.89 - assert(pointer_delta(_top, obj) == word_sz, "Bad undo"); 4.90 - _top = obj; 4.91 - } 4.92 - 4.93 - // The total (word) size of the buffer, including both allocated and 4.94 - // unallocted space. 4.95 - size_t word_sz() { return _word_sz; } 4.96 - 4.97 - // Should only be done if we are about to reset with a new buffer of the 4.98 - // given size. 4.99 - void set_word_size(size_t new_word_sz) { 4.100 - assert(new_word_sz > AlignmentReserve, "Too small"); 4.101 - _word_sz = new_word_sz; 4.102 - } 4.103 - 4.104 - // The number of words of unallocated space remaining in the buffer. 4.105 - size_t words_remaining() { 4.106 - assert(_end >= _top, "Negative buffer"); 4.107 - return pointer_delta(_end, _top, HeapWordSize); 4.108 - } 4.109 - 4.110 - bool contains(void* addr) { 4.111 - return (void*)_bottom <= addr && addr < (void*)_hard_end; 4.112 - } 4.113 - 4.114 - // Sets the space of the buffer to be [buf, space+word_sz()). 4.115 - void set_buf(HeapWord* buf) { 4.116 - _bottom = buf; 4.117 - _top = _bottom; 4.118 - _hard_end = _bottom + word_sz(); 4.119 - _end = _hard_end - AlignmentReserve; 4.120 - assert(_end >= _top, "Negative buffer"); 4.121 - // In support of ergonomic sizing 4.122 - _allocated += word_sz(); 4.123 - } 4.124 - 4.125 - // Flush the stats supporting ergonomic sizing of PLAB's 4.126 - void flush_stats(PLABStats* stats); 4.127 - void flush_stats_and_retire(PLABStats* stats, bool retain) { 4.128 - // We flush the stats first in order to get a reading of 4.129 - // unused space in the last buffer. 4.130 - if (ResizePLAB) { 4.131 - flush_stats(stats); 4.132 - } 4.133 - // Retire the last allocation buffer. 4.134 - retire(true, retain); 4.135 - } 4.136 - 4.137 - // Force future allocations to fail and queries for contains() 4.138 - // to return false 4.139 - void invalidate() { 4.140 - assert(!_retained, "Shouldn't retain an invalidated buffer."); 4.141 - _end = _hard_end; 4.142 - _wasted += pointer_delta(_end, _top); // unused space 4.143 - _top = _end; // force future allocations to fail 4.144 - _bottom = _end; // force future contains() queries to return false 4.145 - } 4.146 - 4.147 - // Fills in the unallocated portion of the buffer with a garbage object. 4.148 - // If "end_of_gc" is TRUE, is after the last use in the GC. IF "retain" 4.149 - // is true, attempt to re-use the unused portion in the next GC. 4.150 - void retire(bool end_of_gc, bool retain); 4.151 - 4.152 - void print() PRODUCT_RETURN; 4.153 -}; 4.154 - 4.155 -// PLAB stats book-keeping 4.156 -class PLABStats VALUE_OBJ_CLASS_SPEC { 4.157 - size_t _allocated; // total allocated 4.158 - size_t _wasted; // of which wasted (internal fragmentation) 4.159 - size_t _unused; // Unused in last buffer 4.160 - size_t _used; // derived = allocated - wasted - unused 4.161 - size_t _desired_plab_sz;// output of filter (below), suitably trimmed and quantized 4.162 - AdaptiveWeightedAverage 4.163 - _filter; // integrator with decay 4.164 - 4.165 - public: 4.166 - PLABStats(size_t desired_plab_sz_, unsigned wt) : 4.167 - _allocated(0), 4.168 - _wasted(0), 4.169 - _unused(0), 4.170 - _used(0), 4.171 - _desired_plab_sz(desired_plab_sz_), 4.172 - _filter(wt) 4.173 - { 4.174 - size_t min_sz = min_size(); 4.175 - size_t max_sz = max_size(); 4.176 - size_t aligned_min_sz = align_object_size(min_sz); 4.177 - size_t aligned_max_sz = align_object_size(max_sz); 4.178 - assert(min_sz <= aligned_min_sz && max_sz >= aligned_max_sz && 4.179 - min_sz <= max_sz, 4.180 - "PLAB clipping computation in adjust_desired_plab_sz()" 4.181 - " may be incorrect"); 4.182 - } 4.183 - 4.184 - static const size_t min_size() { 4.185 - return ParGCAllocBuffer::min_size(); 4.186 - } 4.187 - 4.188 - static const size_t max_size() { 4.189 - return ParGCAllocBuffer::max_size(); 4.190 - } 4.191 - 4.192 - size_t desired_plab_sz() { 4.193 - return _desired_plab_sz; 4.194 - } 4.195 - 4.196 - void adjust_desired_plab_sz(); // filter computation, latches output to 4.197 - // _desired_plab_sz, clears sensor accumulators 4.198 - 4.199 - void add_allocated(size_t v) { 4.200 - Atomic::add_ptr(v, &_allocated); 4.201 - } 4.202 - 4.203 - void add_unused(size_t v) { 4.204 - Atomic::add_ptr(v, &_unused); 4.205 - } 4.206 - 4.207 - void add_wasted(size_t v) { 4.208 - Atomic::add_ptr(v, &_wasted); 4.209 - } 4.210 -}; 4.211 - 4.212 -class ParGCAllocBufferWithBOT: public ParGCAllocBuffer { 4.213 - BlockOffsetArrayContigSpace _bt; 4.214 - BlockOffsetSharedArray* _bsa; 4.215 - HeapWord* _true_end; // end of the whole ParGCAllocBuffer 4.216 - 4.217 - static const size_t ChunkSizeInWords; 4.218 - static const size_t ChunkSizeInBytes; 4.219 - HeapWord* allocate_slow(size_t word_sz); 4.220 - 4.221 - void fill_region_with_block(MemRegion mr, bool contig); 4.222 - 4.223 -public: 4.224 - ParGCAllocBufferWithBOT(size_t word_sz, BlockOffsetSharedArray* bsa); 4.225 - 4.226 - HeapWord* allocate(size_t word_sz) { 4.227 - HeapWord* res = ParGCAllocBuffer::allocate(word_sz); 4.228 - if (res != NULL) { 4.229 - _bt.alloc_block(res, word_sz); 4.230 - } else { 4.231 - res = allocate_slow(word_sz); 4.232 - } 4.233 - return res; 4.234 - } 4.235 - 4.236 - void undo_allocation(HeapWord* obj, size_t word_sz); 4.237 - 4.238 - void set_buf(HeapWord* buf_start) { 4.239 - ParGCAllocBuffer::set_buf(buf_start); 4.240 - _true_end = _hard_end; 4.241 - _bt.set_region(MemRegion(buf_start, word_sz())); 4.242 - _bt.initialize_threshold(); 4.243 - } 4.244 - 4.245 - void retire(bool end_of_gc, bool retain); 4.246 - 4.247 - MemRegion range() { 4.248 - return MemRegion(_top, _true_end); 4.249 - } 4.250 -}; 4.251 - 4.252 -#endif // SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP
5.1 --- a/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp Fri Aug 03 13:24:02 2012 -0700 5.2 +++ b/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp Mon Aug 06 12:20:14 2012 -0700 5.3 @@ -24,11 +24,11 @@ 5.4 5.5 #include "precompiled.hpp" 5.6 #include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp" 5.7 -#include "gc_implementation/parNew/parGCAllocBuffer.hpp" 5.8 #include "gc_implementation/parNew/parNewGeneration.hpp" 5.9 #include "gc_implementation/parNew/parOopClosures.inline.hpp" 5.10 #include "gc_implementation/shared/adaptiveSizePolicy.hpp" 5.11 #include "gc_implementation/shared/ageTable.hpp" 5.12 +#include "gc_implementation/shared/parGCAllocBuffer.hpp" 5.13 #include "gc_implementation/shared/spaceDecorator.hpp" 5.14 #include "memory/defNewGeneration.inline.hpp" 5.15 #include "memory/genCollectedHeap.hpp" 5.16 @@ -453,7 +453,8 @@ 5.17 // retire the last buffer. 5.18 par_scan_state.to_space_alloc_buffer()-> 5.19 flush_stats_and_retire(_gen.plab_stats(), 5.20 - false /* !retain */); 5.21 + true /* end_of_gc */, 5.22 + false /* retain */); 5.23 5.24 // Every thread has its own age table. We need to merge 5.25 // them all into one.
6.1 --- a/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp Fri Aug 03 13:24:02 2012 -0700 6.2 +++ b/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp Mon Aug 06 12:20:14 2012 -0700 6.3 @@ -1,5 +1,5 @@ 6.4 /* 6.5 - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 6.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 6.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6.8 * 6.9 * This code is free software; you can redistribute it and/or modify it 6.10 @@ -25,7 +25,7 @@ 6.11 #ifndef SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARNEWGENERATION_HPP 6.12 #define SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARNEWGENERATION_HPP 6.13 6.14 -#include "gc_implementation/parNew/parGCAllocBuffer.hpp" 6.15 +#include "gc_implementation/shared/parGCAllocBuffer.hpp" 6.16 #include "memory/defNewGeneration.hpp" 6.17 #include "utilities/taskqueue.hpp" 6.18
7.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 7.2 +++ b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.cpp Mon Aug 06 12:20:14 2012 -0700 7.3 @@ -0,0 +1,342 @@ 7.4 +/* 7.5 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 7.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 7.7 + * 7.8 + * This code is free software; you can redistribute it and/or modify it 7.9 + * under the terms of the GNU General Public License version 2 only, as 7.10 + * published by the Free Software Foundation. 7.11 + * 7.12 + * This code is distributed in the hope that it will be useful, but WITHOUT 7.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 7.14 + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 7.15 + * version 2 for more details (a copy is included in the LICENSE file that 7.16 + * accompanied this code). 7.17 + * 7.18 + * You should have received a copy of the GNU General Public License version 7.19 + * 2 along with this work; if not, write to the Free Software Foundation, 7.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 7.21 + * 7.22 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 7.23 + * or visit www.oracle.com if you need additional information or have any 7.24 + * questions. 7.25 + * 7.26 + */ 7.27 + 7.28 +#include "precompiled.hpp" 7.29 +#include "gc_implementation/shared/parGCAllocBuffer.hpp" 7.30 +#include "memory/sharedHeap.hpp" 7.31 +#include "oops/arrayOop.hpp" 7.32 +#include "oops/oop.inline.hpp" 7.33 + 7.34 +ParGCAllocBuffer::ParGCAllocBuffer(size_t desired_plab_sz_) : 7.35 + _word_sz(desired_plab_sz_), _bottom(NULL), _top(NULL), 7.36 + _end(NULL), _hard_end(NULL), 7.37 + _retained(false), _retained_filler(), 7.38 + _allocated(0), _wasted(0) 7.39 +{ 7.40 + assert (min_size() > AlignmentReserve, "Inconsistency!"); 7.41 + // arrayOopDesc::header_size depends on command line initialization. 7.42 + FillerHeaderSize = align_object_size(arrayOopDesc::header_size(T_INT)); 7.43 + AlignmentReserve = oopDesc::header_size() > MinObjAlignment ? FillerHeaderSize : 0; 7.44 +} 7.45 + 7.46 +size_t ParGCAllocBuffer::FillerHeaderSize; 7.47 + 7.48 +// If the minimum object size is greater than MinObjAlignment, we can 7.49 +// end up with a shard at the end of the buffer that's smaller than 7.50 +// the smallest object. We can't allow that because the buffer must 7.51 +// look like it's full of objects when we retire it, so we make 7.52 +// sure we have enough space for a filler int array object. 7.53 +size_t ParGCAllocBuffer::AlignmentReserve; 7.54 + 7.55 +void ParGCAllocBuffer::retire(bool end_of_gc, bool retain) { 7.56 + assert(!retain || end_of_gc, "Can only retain at GC end."); 7.57 + if (_retained) { 7.58 + // If the buffer had been retained shorten the previous filler object. 7.59 + assert(_retained_filler.end() <= _top, "INVARIANT"); 7.60 + CollectedHeap::fill_with_object(_retained_filler); 7.61 + // Wasted space book-keeping, otherwise (normally) done in invalidate() 7.62 + _wasted += _retained_filler.word_size(); 7.63 + _retained = false; 7.64 + } 7.65 + assert(!end_of_gc || !_retained, "At this point, end_of_gc ==> !_retained."); 7.66 + if (_top < _hard_end) { 7.67 + CollectedHeap::fill_with_object(_top, _hard_end); 7.68 + if (!retain) { 7.69 + invalidate(); 7.70 + } else { 7.71 + // Is there wasted space we'd like to retain for the next GC? 7.72 + if (pointer_delta(_end, _top) > FillerHeaderSize) { 7.73 + _retained = true; 7.74 + _retained_filler = MemRegion(_top, FillerHeaderSize); 7.75 + _top = _top + FillerHeaderSize; 7.76 + } else { 7.77 + invalidate(); 7.78 + } 7.79 + } 7.80 + } 7.81 +} 7.82 + 7.83 +void ParGCAllocBuffer::flush_stats(PLABStats* stats) { 7.84 + assert(ResizePLAB, "Wasted work"); 7.85 + stats->add_allocated(_allocated); 7.86 + stats->add_wasted(_wasted); 7.87 + stats->add_unused(pointer_delta(_end, _top)); 7.88 +} 7.89 + 7.90 +// Compute desired plab size and latch result for later 7.91 +// use. This should be called once at the end of parallel 7.92 +// scavenge; it clears the sensor accumulators. 7.93 +void PLABStats::adjust_desired_plab_sz() { 7.94 + assert(ResizePLAB, "Not set"); 7.95 + if (_allocated == 0) { 7.96 + assert(_unused == 0, "Inconsistency in PLAB stats"); 7.97 + _allocated = 1; 7.98 + } 7.99 + double wasted_frac = (double)_unused/(double)_allocated; 7.100 + size_t target_refills = (size_t)((wasted_frac*TargetSurvivorRatio)/ 7.101 + TargetPLABWastePct); 7.102 + if (target_refills == 0) { 7.103 + target_refills = 1; 7.104 + } 7.105 + _used = _allocated - _wasted - _unused; 7.106 + size_t plab_sz = _used/(target_refills*ParallelGCThreads); 7.107 + if (PrintPLAB) gclog_or_tty->print(" (plab_sz = %d ", plab_sz); 7.108 + // Take historical weighted average 7.109 + _filter.sample(plab_sz); 7.110 + // Clip from above and below, and align to object boundary 7.111 + plab_sz = MAX2(min_size(), (size_t)_filter.average()); 7.112 + plab_sz = MIN2(max_size(), plab_sz); 7.113 + plab_sz = align_object_size(plab_sz); 7.114 + // Latch the result 7.115 + if (PrintPLAB) gclog_or_tty->print(" desired_plab_sz = %d) ", plab_sz); 7.116 + _desired_plab_sz = plab_sz; 7.117 + // Now clear the accumulators for next round: 7.118 + // note this needs to be fixed in the case where we 7.119 + // are retaining across scavenges. FIX ME !!! XXX 7.120 + _allocated = 0; 7.121 + _wasted = 0; 7.122 + _unused = 0; 7.123 +} 7.124 + 7.125 +#ifndef PRODUCT 7.126 +void ParGCAllocBuffer::print() { 7.127 + gclog_or_tty->print("parGCAllocBuffer: _bottom: %p _top: %p _end: %p _hard_end: %p" 7.128 + "_retained: %c _retained_filler: [%p,%p)\n", 7.129 + _bottom, _top, _end, _hard_end, 7.130 + "FT"[_retained], _retained_filler.start(), _retained_filler.end()); 7.131 +} 7.132 +#endif // !PRODUCT 7.133 + 7.134 +const size_t ParGCAllocBufferWithBOT::ChunkSizeInWords = 7.135 +MIN2(CardTableModRefBS::par_chunk_heapword_alignment(), 7.136 + ((size_t)Generation::GenGrain)/HeapWordSize); 7.137 +const size_t ParGCAllocBufferWithBOT::ChunkSizeInBytes = 7.138 +MIN2(CardTableModRefBS::par_chunk_heapword_alignment() * HeapWordSize, 7.139 + (size_t)Generation::GenGrain); 7.140 + 7.141 +ParGCAllocBufferWithBOT::ParGCAllocBufferWithBOT(size_t word_sz, 7.142 + BlockOffsetSharedArray* bsa) : 7.143 + ParGCAllocBuffer(word_sz), 7.144 + _bsa(bsa), 7.145 + _bt(bsa, MemRegion(_bottom, _hard_end)), 7.146 + _true_end(_hard_end) 7.147 +{} 7.148 + 7.149 +// The buffer comes with its own BOT, with a shared (obviously) underlying 7.150 +// BlockOffsetSharedArray. We manipulate this BOT in the normal way 7.151 +// as we would for any contiguous space. However, on accasion we 7.152 +// need to do some buffer surgery at the extremities before we 7.153 +// start using the body of the buffer for allocations. Such surgery 7.154 +// (as explained elsewhere) is to prevent allocation on a card that 7.155 +// is in the process of being walked concurrently by another GC thread. 7.156 +// When such surgery happens at a point that is far removed (to the 7.157 +// right of the current allocation point, top), we use the "contig" 7.158 +// parameter below to directly manipulate the shared array without 7.159 +// modifying the _next_threshold state in the BOT. 7.160 +void ParGCAllocBufferWithBOT::fill_region_with_block(MemRegion mr, 7.161 + bool contig) { 7.162 + CollectedHeap::fill_with_object(mr); 7.163 + if (contig) { 7.164 + _bt.alloc_block(mr.start(), mr.end()); 7.165 + } else { 7.166 + _bt.BlockOffsetArray::alloc_block(mr.start(), mr.end()); 7.167 + } 7.168 +} 7.169 + 7.170 +HeapWord* ParGCAllocBufferWithBOT::allocate_slow(size_t word_sz) { 7.171 + HeapWord* res = NULL; 7.172 + if (_true_end > _hard_end) { 7.173 + assert((HeapWord*)align_size_down(intptr_t(_hard_end), 7.174 + ChunkSizeInBytes) == _hard_end, 7.175 + "or else _true_end should be equal to _hard_end"); 7.176 + assert(_retained, "or else _true_end should be equal to _hard_end"); 7.177 + assert(_retained_filler.end() <= _top, "INVARIANT"); 7.178 + CollectedHeap::fill_with_object(_retained_filler); 7.179 + if (_top < _hard_end) { 7.180 + fill_region_with_block(MemRegion(_top, _hard_end), true); 7.181 + } 7.182 + HeapWord* next_hard_end = MIN2(_true_end, _hard_end + ChunkSizeInWords); 7.183 + _retained_filler = MemRegion(_hard_end, FillerHeaderSize); 7.184 + _bt.alloc_block(_retained_filler.start(), _retained_filler.word_size()); 7.185 + _top = _retained_filler.end(); 7.186 + _hard_end = next_hard_end; 7.187 + _end = _hard_end - AlignmentReserve; 7.188 + res = ParGCAllocBuffer::allocate(word_sz); 7.189 + if (res != NULL) { 7.190 + _bt.alloc_block(res, word_sz); 7.191 + } 7.192 + } 7.193 + return res; 7.194 +} 7.195 + 7.196 +void 7.197 +ParGCAllocBufferWithBOT::undo_allocation(HeapWord* obj, size_t word_sz) { 7.198 + ParGCAllocBuffer::undo_allocation(obj, word_sz); 7.199 + // This may back us up beyond the previous threshold, so reset. 7.200 + _bt.set_region(MemRegion(_top, _hard_end)); 7.201 + _bt.initialize_threshold(); 7.202 +} 7.203 + 7.204 +void ParGCAllocBufferWithBOT::retire(bool end_of_gc, bool retain) { 7.205 + assert(!retain || end_of_gc, "Can only retain at GC end."); 7.206 + if (_retained) { 7.207 + // We're about to make the retained_filler into a block. 7.208 + _bt.BlockOffsetArray::alloc_block(_retained_filler.start(), 7.209 + _retained_filler.end()); 7.210 + } 7.211 + // Reset _hard_end to _true_end (and update _end) 7.212 + if (retain && _hard_end != NULL) { 7.213 + assert(_hard_end <= _true_end, "Invariant."); 7.214 + _hard_end = _true_end; 7.215 + _end = MAX2(_top, _hard_end - AlignmentReserve); 7.216 + assert(_end <= _hard_end, "Invariant."); 7.217 + } 7.218 + _true_end = _hard_end; 7.219 + HeapWord* pre_top = _top; 7.220 + 7.221 + ParGCAllocBuffer::retire(end_of_gc, retain); 7.222 + // Now any old _retained_filler is cut back to size, the free part is 7.223 + // filled with a filler object, and top is past the header of that 7.224 + // object. 7.225 + 7.226 + if (retain && _top < _end) { 7.227 + assert(end_of_gc && retain, "Or else retain should be false."); 7.228 + // If the lab does not start on a card boundary, we don't want to 7.229 + // allocate onto that card, since that might lead to concurrent 7.230 + // allocation and card scanning, which we don't support. So we fill 7.231 + // the first card with a garbage object. 7.232 + size_t first_card_index = _bsa->index_for(pre_top); 7.233 + HeapWord* first_card_start = _bsa->address_for_index(first_card_index); 7.234 + if (first_card_start < pre_top) { 7.235 + HeapWord* second_card_start = 7.236 + _bsa->inc_by_region_size(first_card_start); 7.237 + 7.238 + // Ensure enough room to fill with the smallest block 7.239 + second_card_start = MAX2(second_card_start, pre_top + AlignmentReserve); 7.240 + 7.241 + // If the end is already in the first card, don't go beyond it! 7.242 + // Or if the remainder is too small for a filler object, gobble it up. 7.243 + if (_hard_end < second_card_start || 7.244 + pointer_delta(_hard_end, second_card_start) < AlignmentReserve) { 7.245 + second_card_start = _hard_end; 7.246 + } 7.247 + if (pre_top < second_card_start) { 7.248 + MemRegion first_card_suffix(pre_top, second_card_start); 7.249 + fill_region_with_block(first_card_suffix, true); 7.250 + } 7.251 + pre_top = second_card_start; 7.252 + _top = pre_top; 7.253 + _end = MAX2(_top, _hard_end - AlignmentReserve); 7.254 + } 7.255 + 7.256 + // If the lab does not end on a card boundary, we don't want to 7.257 + // allocate onto that card, since that might lead to concurrent 7.258 + // allocation and card scanning, which we don't support. So we fill 7.259 + // the last card with a garbage object. 7.260 + size_t last_card_index = _bsa->index_for(_hard_end); 7.261 + HeapWord* last_card_start = _bsa->address_for_index(last_card_index); 7.262 + if (last_card_start < _hard_end) { 7.263 + 7.264 + // Ensure enough room to fill with the smallest block 7.265 + last_card_start = MIN2(last_card_start, _hard_end - AlignmentReserve); 7.266 + 7.267 + // If the top is already in the last card, don't go back beyond it! 7.268 + // Or if the remainder is too small for a filler object, gobble it up. 7.269 + if (_top > last_card_start || 7.270 + pointer_delta(last_card_start, _top) < AlignmentReserve) { 7.271 + last_card_start = _top; 7.272 + } 7.273 + if (last_card_start < _hard_end) { 7.274 + MemRegion last_card_prefix(last_card_start, _hard_end); 7.275 + fill_region_with_block(last_card_prefix, false); 7.276 + } 7.277 + _hard_end = last_card_start; 7.278 + _end = MAX2(_top, _hard_end - AlignmentReserve); 7.279 + _true_end = _hard_end; 7.280 + assert(_end <= _hard_end, "Invariant."); 7.281 + } 7.282 + 7.283 + // At this point: 7.284 + // 1) we had a filler object from the original top to hard_end. 7.285 + // 2) We've filled in any partial cards at the front and back. 7.286 + if (pre_top < _hard_end) { 7.287 + // Now we can reset the _bt to do allocation in the given area. 7.288 + MemRegion new_filler(pre_top, _hard_end); 7.289 + fill_region_with_block(new_filler, false); 7.290 + _top = pre_top + ParGCAllocBuffer::FillerHeaderSize; 7.291 + // If there's no space left, don't retain. 7.292 + if (_top >= _end) { 7.293 + _retained = false; 7.294 + invalidate(); 7.295 + return; 7.296 + } 7.297 + _retained_filler = MemRegion(pre_top, _top); 7.298 + _bt.set_region(MemRegion(_top, _hard_end)); 7.299 + _bt.initialize_threshold(); 7.300 + assert(_bt.threshold() > _top, "initialize_threshold failed!"); 7.301 + 7.302 + // There may be other reasons for queries into the middle of the 7.303 + // filler object. When such queries are done in parallel with 7.304 + // allocation, bad things can happen, if the query involves object 7.305 + // iteration. So we ensure that such queries do not involve object 7.306 + // iteration, by putting another filler object on the boundaries of 7.307 + // such queries. One such is the object spanning a parallel card 7.308 + // chunk boundary. 7.309 + 7.310 + // "chunk_boundary" is the address of the first chunk boundary less 7.311 + // than "hard_end". 7.312 + HeapWord* chunk_boundary = 7.313 + (HeapWord*)align_size_down(intptr_t(_hard_end-1), ChunkSizeInBytes); 7.314 + assert(chunk_boundary < _hard_end, "Or else above did not work."); 7.315 + assert(pointer_delta(_true_end, chunk_boundary) >= AlignmentReserve, 7.316 + "Consequence of last card handling above."); 7.317 + 7.318 + if (_top <= chunk_boundary) { 7.319 + assert(_true_end == _hard_end, "Invariant."); 7.320 + while (_top <= chunk_boundary) { 7.321 + assert(pointer_delta(_hard_end, chunk_boundary) >= AlignmentReserve, 7.322 + "Consequence of last card handling above."); 7.323 + _bt.BlockOffsetArray::alloc_block(chunk_boundary, _hard_end); 7.324 + CollectedHeap::fill_with_object(chunk_boundary, _hard_end); 7.325 + _hard_end = chunk_boundary; 7.326 + chunk_boundary -= ChunkSizeInWords; 7.327 + } 7.328 + _end = _hard_end - AlignmentReserve; 7.329 + assert(_top <= _end, "Invariant."); 7.330 + // Now reset the initial filler chunk so it doesn't overlap with 7.331 + // the one(s) inserted above. 7.332 + MemRegion new_filler(pre_top, _hard_end); 7.333 + fill_region_with_block(new_filler, false); 7.334 + } 7.335 + } else { 7.336 + _retained = false; 7.337 + invalidate(); 7.338 + } 7.339 + } else { 7.340 + assert(!end_of_gc || 7.341 + (!_retained && _true_end == _hard_end), "Checking."); 7.342 + } 7.343 + assert(_end <= _hard_end, "Invariant."); 7.344 + assert(_top < _end || _top == _hard_end, "Invariant"); 7.345 +}
8.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 8.2 +++ b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp Mon Aug 06 12:20:14 2012 -0700 8.3 @@ -0,0 +1,249 @@ 8.4 +/* 8.5 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 8.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 8.7 + * 8.8 + * This code is free software; you can redistribute it and/or modify it 8.9 + * under the terms of the GNU General Public License version 2 only, as 8.10 + * published by the Free Software Foundation. 8.11 + * 8.12 + * This code is distributed in the hope that it will be useful, but WITHOUT 8.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 8.14 + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 8.15 + * version 2 for more details (a copy is included in the LICENSE file that 8.16 + * accompanied this code). 8.17 + * 8.18 + * You should have received a copy of the GNU General Public License version 8.19 + * 2 along with this work; if not, write to the Free Software Foundation, 8.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 8.21 + * 8.22 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 8.23 + * or visit www.oracle.com if you need additional information or have any 8.24 + * questions. 8.25 + * 8.26 + */ 8.27 + 8.28 +#ifndef SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP 8.29 +#define SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP 8.30 + 8.31 +#include "memory/allocation.hpp" 8.32 +#include "memory/blockOffsetTable.hpp" 8.33 +#include "memory/threadLocalAllocBuffer.hpp" 8.34 +#include "utilities/globalDefinitions.hpp" 8.35 + 8.36 +// Forward decl. 8.37 + 8.38 +class PLABStats; 8.39 + 8.40 +// A per-thread allocation buffer used during GC. 8.41 +class ParGCAllocBuffer: public CHeapObj<mtGC> { 8.42 +protected: 8.43 + char head[32]; 8.44 + size_t _word_sz; // in HeapWord units 8.45 + HeapWord* _bottom; 8.46 + HeapWord* _top; 8.47 + HeapWord* _end; // last allocatable address + 1 8.48 + HeapWord* _hard_end; // _end + AlignmentReserve 8.49 + bool _retained; // whether we hold a _retained_filler 8.50 + MemRegion _retained_filler; 8.51 + // In support of ergonomic sizing of PLAB's 8.52 + size_t _allocated; // in HeapWord units 8.53 + size_t _wasted; // in HeapWord units 8.54 + char tail[32]; 8.55 + static size_t FillerHeaderSize; 8.56 + static size_t AlignmentReserve; 8.57 + 8.58 +public: 8.59 + // Initializes the buffer to be empty, but with the given "word_sz". 8.60 + // Must get initialized with "set_buf" for an allocation to succeed. 8.61 + ParGCAllocBuffer(size_t word_sz); 8.62 + 8.63 + static const size_t min_size() { 8.64 + return ThreadLocalAllocBuffer::min_size(); 8.65 + } 8.66 + 8.67 + static const size_t max_size() { 8.68 + return ThreadLocalAllocBuffer::max_size(); 8.69 + } 8.70 + 8.71 + // If an allocation of the given "word_sz" can be satisfied within the 8.72 + // buffer, do the allocation, returning a pointer to the start of the 8.73 + // allocated block. If the allocation request cannot be satisfied, 8.74 + // return NULL. 8.75 + HeapWord* allocate(size_t word_sz) { 8.76 + HeapWord* res = _top; 8.77 + if (pointer_delta(_end, _top) >= word_sz) { 8.78 + _top = _top + word_sz; 8.79 + return res; 8.80 + } else { 8.81 + return NULL; 8.82 + } 8.83 + } 8.84 + 8.85 + // Undo the last allocation in the buffer, which is required to be of the 8.86 + // "obj" of the given "word_sz". 8.87 + void undo_allocation(HeapWord* obj, size_t word_sz) { 8.88 + assert(pointer_delta(_top, _bottom) >= word_sz, "Bad undo"); 8.89 + assert(pointer_delta(_top, obj) == word_sz, "Bad undo"); 8.90 + _top = obj; 8.91 + } 8.92 + 8.93 + // The total (word) size of the buffer, including both allocated and 8.94 + // unallocted space. 8.95 + size_t word_sz() { return _word_sz; } 8.96 + 8.97 + // Should only be done if we are about to reset with a new buffer of the 8.98 + // given size. 8.99 + void set_word_size(size_t new_word_sz) { 8.100 + assert(new_word_sz > AlignmentReserve, "Too small"); 8.101 + _word_sz = new_word_sz; 8.102 + } 8.103 + 8.104 + // The number of words of unallocated space remaining in the buffer. 8.105 + size_t words_remaining() { 8.106 + assert(_end >= _top, "Negative buffer"); 8.107 + return pointer_delta(_end, _top, HeapWordSize); 8.108 + } 8.109 + 8.110 + bool contains(void* addr) { 8.111 + return (void*)_bottom <= addr && addr < (void*)_hard_end; 8.112 + } 8.113 + 8.114 + // Sets the space of the buffer to be [buf, space+word_sz()). 8.115 + void set_buf(HeapWord* buf) { 8.116 + _bottom = buf; 8.117 + _top = _bottom; 8.118 + _hard_end = _bottom + word_sz(); 8.119 + _end = _hard_end - AlignmentReserve; 8.120 + assert(_end >= _top, "Negative buffer"); 8.121 + // In support of ergonomic sizing 8.122 + _allocated += word_sz(); 8.123 + } 8.124 + 8.125 + // Flush the stats supporting ergonomic sizing of PLAB's 8.126 + void flush_stats(PLABStats* stats); 8.127 + void flush_stats_and_retire(PLABStats* stats, bool end_of_gc, bool retain) { 8.128 + // We flush the stats first in order to get a reading of 8.129 + // unused space in the last buffer. 8.130 + if (ResizePLAB) { 8.131 + flush_stats(stats); 8.132 + } 8.133 + // Retire the last allocation buffer. 8.134 + retire(end_of_gc, retain); 8.135 + } 8.136 + 8.137 + // Force future allocations to fail and queries for contains() 8.138 + // to return false 8.139 + void invalidate() { 8.140 + assert(!_retained, "Shouldn't retain an invalidated buffer."); 8.141 + _end = _hard_end; 8.142 + _wasted += pointer_delta(_end, _top); // unused space 8.143 + _top = _end; // force future allocations to fail 8.144 + _bottom = _end; // force future contains() queries to return false 8.145 + } 8.146 + 8.147 + // Fills in the unallocated portion of the buffer with a garbage object. 8.148 + // If "end_of_gc" is TRUE, is after the last use in the GC. IF "retain" 8.149 + // is true, attempt to re-use the unused portion in the next GC. 8.150 + void retire(bool end_of_gc, bool retain); 8.151 + 8.152 + void print() PRODUCT_RETURN; 8.153 +}; 8.154 + 8.155 +// PLAB stats book-keeping 8.156 +class PLABStats VALUE_OBJ_CLASS_SPEC { 8.157 + size_t _allocated; // total allocated 8.158 + size_t _wasted; // of which wasted (internal fragmentation) 8.159 + size_t _unused; // Unused in last buffer 8.160 + size_t _used; // derived = allocated - wasted - unused 8.161 + size_t _desired_plab_sz;// output of filter (below), suitably trimmed and quantized 8.162 + AdaptiveWeightedAverage 8.163 + _filter; // integrator with decay 8.164 + 8.165 + public: 8.166 + PLABStats(size_t desired_plab_sz_, unsigned wt) : 8.167 + _allocated(0), 8.168 + _wasted(0), 8.169 + _unused(0), 8.170 + _used(0), 8.171 + _desired_plab_sz(desired_plab_sz_), 8.172 + _filter(wt) 8.173 + { 8.174 + size_t min_sz = min_size(); 8.175 + size_t max_sz = max_size(); 8.176 + size_t aligned_min_sz = align_object_size(min_sz); 8.177 + size_t aligned_max_sz = align_object_size(max_sz); 8.178 + assert(min_sz <= aligned_min_sz && max_sz >= aligned_max_sz && 8.179 + min_sz <= max_sz, 8.180 + "PLAB clipping computation in adjust_desired_plab_sz()" 8.181 + " may be incorrect"); 8.182 + } 8.183 + 8.184 + static const size_t min_size() { 8.185 + return ParGCAllocBuffer::min_size(); 8.186 + } 8.187 + 8.188 + static const size_t max_size() { 8.189 + return ParGCAllocBuffer::max_size(); 8.190 + } 8.191 + 8.192 + size_t desired_plab_sz() { 8.193 + return _desired_plab_sz; 8.194 + } 8.195 + 8.196 + void adjust_desired_plab_sz(); // filter computation, latches output to 8.197 + // _desired_plab_sz, clears sensor accumulators 8.198 + 8.199 + void add_allocated(size_t v) { 8.200 + Atomic::add_ptr(v, &_allocated); 8.201 + } 8.202 + 8.203 + void add_unused(size_t v) { 8.204 + Atomic::add_ptr(v, &_unused); 8.205 + } 8.206 + 8.207 + void add_wasted(size_t v) { 8.208 + Atomic::add_ptr(v, &_wasted); 8.209 + } 8.210 +}; 8.211 + 8.212 +class ParGCAllocBufferWithBOT: public ParGCAllocBuffer { 8.213 + BlockOffsetArrayContigSpace _bt; 8.214 + BlockOffsetSharedArray* _bsa; 8.215 + HeapWord* _true_end; // end of the whole ParGCAllocBuffer 8.216 + 8.217 + static const size_t ChunkSizeInWords; 8.218 + static const size_t ChunkSizeInBytes; 8.219 + HeapWord* allocate_slow(size_t word_sz); 8.220 + 8.221 + void fill_region_with_block(MemRegion mr, bool contig); 8.222 + 8.223 +public: 8.224 + ParGCAllocBufferWithBOT(size_t word_sz, BlockOffsetSharedArray* bsa); 8.225 + 8.226 + HeapWord* allocate(size_t word_sz) { 8.227 + HeapWord* res = ParGCAllocBuffer::allocate(word_sz); 8.228 + if (res != NULL) { 8.229 + _bt.alloc_block(res, word_sz); 8.230 + } else { 8.231 + res = allocate_slow(word_sz); 8.232 + } 8.233 + return res; 8.234 + } 8.235 + 8.236 + void undo_allocation(HeapWord* obj, size_t word_sz); 8.237 + 8.238 + void set_buf(HeapWord* buf_start) { 8.239 + ParGCAllocBuffer::set_buf(buf_start); 8.240 + _true_end = _hard_end; 8.241 + _bt.set_region(MemRegion(buf_start, word_sz())); 8.242 + _bt.initialize_threshold(); 8.243 + } 8.244 + 8.245 + void retire(bool end_of_gc, bool retain); 8.246 + 8.247 + MemRegion range() { 8.248 + return MemRegion(_top, _true_end); 8.249 + } 8.250 +}; 8.251 + 8.252 +#endif // SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP
9.1 --- a/src/share/vm/memory/tenuredGeneration.cpp Fri Aug 03 13:24:02 2012 -0700 9.2 +++ b/src/share/vm/memory/tenuredGeneration.cpp Mon Aug 06 12:20:14 2012 -0700 9.3 @@ -1,5 +1,5 @@ 9.4 /* 9.5 - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 9.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 9.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 9.8 * 9.9 * This code is free software; you can redistribute it and/or modify it 9.10 @@ -23,8 +23,8 @@ 9.11 */ 9.12 9.13 #include "precompiled.hpp" 9.14 -#include "gc_implementation/parNew/parGCAllocBuffer.hpp" 9.15 #include "gc_implementation/shared/collectorCounters.hpp" 9.16 +#include "gc_implementation/shared/parGCAllocBuffer.hpp" 9.17 #include "memory/allocation.inline.hpp" 9.18 #include "memory/blockOffsetTable.inline.hpp" 9.19 #include "memory/generation.inline.hpp"
10.1 --- a/src/share/vm/precompiled/precompiled.hpp Fri Aug 03 13:24:02 2012 -0700 10.2 +++ b/src/share/vm/precompiled/precompiled.hpp Mon Aug 06 12:20:14 2012 -0700 10.3 @@ -1,5 +1,5 @@ 10.4 /* 10.5 - * Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved. 10.6 + * Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. 10.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 10.8 * 10.9 * This code is free software; you can redistribute it and/or modify it 10.10 @@ -306,7 +306,6 @@ 10.11 # include "gc_implementation/g1/g1_specialized_oop_closures.hpp" 10.12 # include "gc_implementation/g1/ptrQueue.hpp" 10.13 # include "gc_implementation/g1/satbQueue.hpp" 10.14 -# include "gc_implementation/parNew/parGCAllocBuffer.hpp" 10.15 # include "gc_implementation/parNew/parOopClosures.hpp" 10.16 # include "gc_implementation/parallelScavenge/objectStartArray.hpp" 10.17 # include "gc_implementation/parallelScavenge/parMarkBitMap.hpp" 10.18 @@ -322,6 +321,7 @@ 10.19 # include "gc_implementation/parallelScavenge/psYoungGen.hpp" 10.20 # include "gc_implementation/shared/gcAdaptivePolicyCounters.hpp" 10.21 # include "gc_implementation/shared/gcPolicyCounters.hpp" 10.22 +# include "gc_implementation/shared/parGCAllocBuffer.hpp" 10.23 #endif // SERIALGC 10.24 10.25 #endif // !DONT_USE_PRECOMPILED_HEADER