Thu, 22 Sep 2011 10:57:37 -0700
6484982: G1: process references during evacuation pauses
Summary: G1 now uses two reference processors - one is used by concurrent marking and the other is used by STW GCs (both full and incremental evacuation pauses). In an evacuation pause, the reference processor is embedded into the closures used to scan objects. Doing so causes causes reference objects to be 'discovered' by the reference processor. At the end of the evacuation pause, these discovered reference objects are processed - preserving (and copying) referent objects (and their reachable graphs) as appropriate.
Reviewed-by: ysr, jwilhelm, brutisso, stefank, tonyp
1 /*
2 * Copyright (c) 2002, 2011, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
25 #include "precompiled.hpp"
26 #include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp"
27 #include "gc_implementation/parallelScavenge/psOldGen.hpp"
28 #include "gc_implementation/parallelScavenge/psPromotionManager.inline.hpp"
29 #include "gc_implementation/parallelScavenge/psScavenge.inline.hpp"
30 #include "gc_implementation/shared/mutableSpace.hpp"
31 #include "memory/memRegion.hpp"
32 #include "oops/oop.inline.hpp"
33 #include "oops/oop.psgc.inline.hpp"
35 PSPromotionManager** PSPromotionManager::_manager_array = NULL;
36 OopStarTaskQueueSet* PSPromotionManager::_stack_array_depth = NULL;
37 PSOldGen* PSPromotionManager::_old_gen = NULL;
38 MutableSpace* PSPromotionManager::_young_space = NULL;
40 void PSPromotionManager::initialize() {
41 ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap();
42 assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Sanity");
44 _old_gen = heap->old_gen();
45 _young_space = heap->young_gen()->to_space();
47 assert(_manager_array == NULL, "Attempt to initialize twice");
48 _manager_array = NEW_C_HEAP_ARRAY(PSPromotionManager*, ParallelGCThreads+1 );
49 guarantee(_manager_array != NULL, "Could not initialize promotion manager");
51 _stack_array_depth = new OopStarTaskQueueSet(ParallelGCThreads);
52 guarantee(_stack_array_depth != NULL, "Cound not initialize promotion manager");
54 // Create and register the PSPromotionManager(s) for the worker threads.
55 for(uint i=0; i<ParallelGCThreads; i++) {
56 _manager_array[i] = new PSPromotionManager();
57 guarantee(_manager_array[i] != NULL, "Could not create PSPromotionManager");
58 stack_array_depth()->register_queue(i, _manager_array[i]->claimed_stack_depth());
59 }
61 // The VMThread gets its own PSPromotionManager, which is not available
62 // for work stealing.
63 _manager_array[ParallelGCThreads] = new PSPromotionManager();
64 guarantee(_manager_array[ParallelGCThreads] != NULL, "Could not create PSPromotionManager");
65 }
67 PSPromotionManager* PSPromotionManager::gc_thread_promotion_manager(int index) {
68 assert(index >= 0 && index < (int)ParallelGCThreads, "index out of range");
69 assert(_manager_array != NULL, "Sanity");
70 return _manager_array[index];
71 }
73 PSPromotionManager* PSPromotionManager::vm_thread_promotion_manager() {
74 assert(_manager_array != NULL, "Sanity");
75 return _manager_array[ParallelGCThreads];
76 }
78 void PSPromotionManager::pre_scavenge() {
79 ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap();
80 assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Sanity");
82 _young_space = heap->young_gen()->to_space();
84 for(uint i=0; i<ParallelGCThreads+1; i++) {
85 manager_array(i)->reset();
86 }
87 }
89 void PSPromotionManager::post_scavenge() {
90 TASKQUEUE_STATS_ONLY(if (PrintGCDetails && ParallelGCVerbose) print_stats());
91 for (uint i = 0; i < ParallelGCThreads + 1; i++) {
92 PSPromotionManager* manager = manager_array(i);
93 assert(manager->claimed_stack_depth()->is_empty(), "should be empty");
94 manager->flush_labs();
95 }
96 }
98 #if TASKQUEUE_STATS
99 void
100 PSPromotionManager::print_taskqueue_stats(uint i) const {
101 tty->print("%3u ", i);
102 _claimed_stack_depth.stats.print();
103 tty->cr();
104 }
106 void
107 PSPromotionManager::print_local_stats(uint i) const {
108 #define FMT " " SIZE_FORMAT_W(10)
109 tty->print_cr("%3u" FMT FMT FMT FMT, i, _masked_pushes, _masked_steals,
110 _arrays_chunked, _array_chunks_processed);
111 #undef FMT
112 }
114 static const char* const pm_stats_hdr[] = {
115 " --------masked------- arrays array",
116 "thr push steal chunked chunks",
117 "--- ---------- ---------- ---------- ----------"
118 };
120 void
121 PSPromotionManager::print_stats() {
122 tty->print_cr("== GC Tasks Stats, GC %3d",
123 Universe::heap()->total_collections());
125 tty->print("thr "); TaskQueueStats::print_header(1); tty->cr();
126 tty->print("--- "); TaskQueueStats::print_header(2); tty->cr();
127 for (uint i = 0; i < ParallelGCThreads + 1; ++i) {
128 manager_array(i)->print_taskqueue_stats(i);
129 }
131 const uint hlines = sizeof(pm_stats_hdr) / sizeof(pm_stats_hdr[0]);
132 for (uint i = 0; i < hlines; ++i) tty->print_cr(pm_stats_hdr[i]);
133 for (uint i = 0; i < ParallelGCThreads + 1; ++i) {
134 manager_array(i)->print_local_stats(i);
135 }
136 }
138 void
139 PSPromotionManager::reset_stats() {
140 claimed_stack_depth()->stats.reset();
141 _masked_pushes = _masked_steals = 0;
142 _arrays_chunked = _array_chunks_processed = 0;
143 }
144 #endif // TASKQUEUE_STATS
146 PSPromotionManager::PSPromotionManager() {
147 ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap();
148 assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Sanity");
150 // We set the old lab's start array.
151 _old_lab.set_start_array(old_gen()->start_array());
153 uint queue_size;
154 claimed_stack_depth()->initialize();
155 queue_size = claimed_stack_depth()->max_elems();
157 _totally_drain = (ParallelGCThreads == 1) || (GCDrainStackTargetSize == 0);
158 if (_totally_drain) {
159 _target_stack_size = 0;
160 } else {
161 // don't let the target stack size to be more than 1/4 of the entries
162 _target_stack_size = (uint) MIN2((uint) GCDrainStackTargetSize,
163 (uint) (queue_size / 4));
164 }
166 _array_chunk_size = ParGCArrayScanChunk;
167 // let's choose 1.5x the chunk size
168 _min_array_size_for_chunking = 3 * _array_chunk_size / 2;
170 reset();
171 }
173 void PSPromotionManager::reset() {
174 assert(stacks_empty(), "reset of non-empty stack");
176 // We need to get an assert in here to make sure the labs are always flushed.
178 ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap();
179 assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Sanity");
181 // Do not prefill the LAB's, save heap wastage!
182 HeapWord* lab_base = young_space()->top();
183 _young_lab.initialize(MemRegion(lab_base, (size_t)0));
184 _young_gen_is_full = false;
186 lab_base = old_gen()->object_space()->top();
187 _old_lab.initialize(MemRegion(lab_base, (size_t)0));
188 _old_gen_is_full = false;
190 TASKQUEUE_STATS_ONLY(reset_stats());
191 }
194 void PSPromotionManager::drain_stacks_depth(bool totally_drain) {
195 totally_drain = totally_drain || _totally_drain;
197 #ifdef ASSERT
198 ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap();
199 assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Sanity");
200 MutableSpace* to_space = heap->young_gen()->to_space();
201 MutableSpace* old_space = heap->old_gen()->object_space();
202 MutableSpace* perm_space = heap->perm_gen()->object_space();
203 #endif /* ASSERT */
205 OopStarTaskQueue* const tq = claimed_stack_depth();
206 do {
207 StarTask p;
209 // Drain overflow stack first, so other threads can steal from
210 // claimed stack while we work.
211 while (tq->pop_overflow(p)) {
212 process_popped_location_depth(p);
213 }
215 if (totally_drain) {
216 while (tq->pop_local(p)) {
217 process_popped_location_depth(p);
218 }
219 } else {
220 while (tq->size() > _target_stack_size && tq->pop_local(p)) {
221 process_popped_location_depth(p);
222 }
223 }
224 } while (totally_drain && !tq->taskqueue_empty() || !tq->overflow_empty());
226 assert(!totally_drain || tq->taskqueue_empty(), "Sanity");
227 assert(totally_drain || tq->size() <= _target_stack_size, "Sanity");
228 assert(tq->overflow_empty(), "Sanity");
229 }
231 void PSPromotionManager::flush_labs() {
232 assert(stacks_empty(), "Attempt to flush lab with live stack");
234 // If either promotion lab fills up, we can flush the
235 // lab but not refill it, so check first.
236 assert(!_young_lab.is_flushed() || _young_gen_is_full, "Sanity");
237 if (!_young_lab.is_flushed())
238 _young_lab.flush();
240 assert(!_old_lab.is_flushed() || _old_gen_is_full, "Sanity");
241 if (!_old_lab.is_flushed())
242 _old_lab.flush();
244 // Let PSScavenge know if we overflowed
245 if (_young_gen_is_full) {
246 PSScavenge::set_survivor_overflow(true);
247 }
248 }
250 //
251 // This method is pretty bulky. It would be nice to split it up
252 // into smaller submethods, but we need to be careful not to hurt
253 // performance.
254 //
256 oop PSPromotionManager::copy_to_survivor_space(oop o) {
257 assert(PSScavenge::should_scavenge(&o), "Sanity");
259 oop new_obj = NULL;
261 // NOTE! We must be very careful with any methods that access the mark
262 // in o. There may be multiple threads racing on it, and it may be forwarded
263 // at any time. Do not use oop methods for accessing the mark!
264 markOop test_mark = o->mark();
266 // The same test as "o->is_forwarded()"
267 if (!test_mark->is_marked()) {
268 bool new_obj_is_tenured = false;
269 size_t new_obj_size = o->size();
271 // Find the objects age, MT safe.
272 int age = (test_mark->has_displaced_mark_helper() /* o->has_displaced_mark() */) ?
273 test_mark->displaced_mark_helper()->age() : test_mark->age();
275 // Try allocating obj in to-space (unless too old)
276 if (age < PSScavenge::tenuring_threshold()) {
277 new_obj = (oop) _young_lab.allocate(new_obj_size);
278 if (new_obj == NULL && !_young_gen_is_full) {
279 // Do we allocate directly, or flush and refill?
280 if (new_obj_size > (YoungPLABSize / 2)) {
281 // Allocate this object directly
282 new_obj = (oop)young_space()->cas_allocate(new_obj_size);
283 } else {
284 // Flush and fill
285 _young_lab.flush();
287 HeapWord* lab_base = young_space()->cas_allocate(YoungPLABSize);
288 if (lab_base != NULL) {
289 _young_lab.initialize(MemRegion(lab_base, YoungPLABSize));
290 // Try the young lab allocation again.
291 new_obj = (oop) _young_lab.allocate(new_obj_size);
292 } else {
293 _young_gen_is_full = true;
294 }
295 }
296 }
297 }
299 // Otherwise try allocating obj tenured
300 if (new_obj == NULL) {
301 #ifndef PRODUCT
302 if (Universe::heap()->promotion_should_fail()) {
303 return oop_promotion_failed(o, test_mark);
304 }
305 #endif // #ifndef PRODUCT
307 new_obj = (oop) _old_lab.allocate(new_obj_size);
308 new_obj_is_tenured = true;
310 if (new_obj == NULL) {
311 if (!_old_gen_is_full) {
312 // Do we allocate directly, or flush and refill?
313 if (new_obj_size > (OldPLABSize / 2)) {
314 // Allocate this object directly
315 new_obj = (oop)old_gen()->cas_allocate(new_obj_size);
316 } else {
317 // Flush and fill
318 _old_lab.flush();
320 HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize);
321 if(lab_base != NULL) {
322 _old_lab.initialize(MemRegion(lab_base, OldPLABSize));
323 // Try the old lab allocation again.
324 new_obj = (oop) _old_lab.allocate(new_obj_size);
325 }
326 }
327 }
329 // This is the promotion failed test, and code handling.
330 // The code belongs here for two reasons. It is slightly
331 // different thatn the code below, and cannot share the
332 // CAS testing code. Keeping the code here also minimizes
333 // the impact on the common case fast path code.
335 if (new_obj == NULL) {
336 _old_gen_is_full = true;
337 return oop_promotion_failed(o, test_mark);
338 }
339 }
340 }
342 assert(new_obj != NULL, "allocation should have succeeded");
344 // Copy obj
345 Copy::aligned_disjoint_words((HeapWord*)o, (HeapWord*)new_obj, new_obj_size);
347 // Now we have to CAS in the header.
348 if (o->cas_forward_to(new_obj, test_mark)) {
349 // We won any races, we "own" this object.
350 assert(new_obj == o->forwardee(), "Sanity");
352 // Increment age if obj still in new generation. Now that
353 // we're dealing with a markOop that cannot change, it is
354 // okay to use the non mt safe oop methods.
355 if (!new_obj_is_tenured) {
356 new_obj->incr_age();
357 assert(young_space()->contains(new_obj), "Attempt to push non-promoted obj");
358 }
360 // Do the size comparison first with new_obj_size, which we
361 // already have. Hopefully, only a few objects are larger than
362 // _min_array_size_for_chunking, and most of them will be arrays.
363 // So, the is->objArray() test would be very infrequent.
364 if (new_obj_size > _min_array_size_for_chunking &&
365 new_obj->is_objArray() &&
366 PSChunkLargeArrays) {
367 // we'll chunk it
368 oop* const masked_o = mask_chunked_array_oop(o);
369 push_depth(masked_o);
370 TASKQUEUE_STATS_ONLY(++_arrays_chunked; ++_masked_pushes);
371 } else {
372 // we'll just push its contents
373 new_obj->push_contents(this);
374 }
375 } else {
376 // We lost, someone else "owns" this object
377 guarantee(o->is_forwarded(), "Object must be forwarded if the cas failed.");
379 // Try to deallocate the space. If it was directly allocated we cannot
380 // deallocate it, so we have to test. If the deallocation fails,
381 // overwrite with a filler object.
382 if (new_obj_is_tenured) {
383 if (!_old_lab.unallocate_object(new_obj)) {
384 CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size);
385 }
386 } else if (!_young_lab.unallocate_object(new_obj)) {
387 CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size);
388 }
390 // don't update this before the unallocation!
391 new_obj = o->forwardee();
392 }
393 } else {
394 assert(o->is_forwarded(), "Sanity");
395 new_obj = o->forwardee();
396 }
398 #ifdef DEBUG
399 // This code must come after the CAS test, or it will print incorrect
400 // information.
401 if (TraceScavenge) {
402 gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " -> " PTR_FORMAT " (" SIZE_FORMAT ")}",
403 PSScavenge::should_scavenge(&new_obj) ? "copying" : "tenuring",
404 new_obj->blueprint()->internal_name(), o, new_obj, new_obj->size());
405 }
406 #endif
408 return new_obj;
409 }
411 template <class T> void PSPromotionManager::process_array_chunk_work(
412 oop obj,
413 int start, int end) {
414 assert(start <= end, "invariant");
415 T* const base = (T*)objArrayOop(obj)->base();
416 T* p = base + start;
417 T* const chunk_end = base + end;
418 while (p < chunk_end) {
419 if (PSScavenge::should_scavenge(p)) {
420 claim_or_forward_depth(p);
421 }
422 ++p;
423 }
424 }
426 void PSPromotionManager::process_array_chunk(oop old) {
427 assert(PSChunkLargeArrays, "invariant");
428 assert(old->is_objArray(), "invariant");
429 assert(old->is_forwarded(), "invariant");
431 TASKQUEUE_STATS_ONLY(++_array_chunks_processed);
433 oop const obj = old->forwardee();
435 int start;
436 int const end = arrayOop(old)->length();
437 if (end > (int) _min_array_size_for_chunking) {
438 // we'll chunk more
439 start = end - _array_chunk_size;
440 assert(start > 0, "invariant");
441 arrayOop(old)->set_length(start);
442 push_depth(mask_chunked_array_oop(old));
443 TASKQUEUE_STATS_ONLY(++_masked_pushes);
444 } else {
445 // this is the final chunk for this array
446 start = 0;
447 int const actual_length = arrayOop(obj)->length();
448 arrayOop(old)->set_length(actual_length);
449 }
451 if (UseCompressedOops) {
452 process_array_chunk_work<narrowOop>(obj, start, end);
453 } else {
454 process_array_chunk_work<oop>(obj, start, end);
455 }
456 }
458 oop PSPromotionManager::oop_promotion_failed(oop obj, markOop obj_mark) {
459 assert(_old_gen_is_full || PromotionFailureALot, "Sanity");
461 // Attempt to CAS in the header.
462 // This tests if the header is still the same as when
463 // this started. If it is the same (i.e., no forwarding
464 // pointer has been installed), then this thread owns
465 // it.
466 if (obj->cas_forward_to(obj, obj_mark)) {
467 // We won any races, we "own" this object.
468 assert(obj == obj->forwardee(), "Sanity");
470 obj->push_contents(this);
472 // Save the mark if needed
473 PSScavenge::oop_promotion_failed(obj, obj_mark);
474 } else {
475 // We lost, someone else "owns" this object
476 guarantee(obj->is_forwarded(), "Object must be forwarded if the cas failed.");
478 // No unallocation to worry about.
479 obj = obj->forwardee();
480 }
482 #ifdef DEBUG
483 if (TraceScavenge) {
484 gclog_or_tty->print_cr("{%s %s 0x%x (%d)}",
485 "promotion-failure",
486 obj->blueprint()->internal_name(),
487 obj, obj->size());
489 }
490 #endif
492 return obj;
493 }