Sat, 06 Oct 2012 01:17:44 -0700
7127708: G1: change task num types from int to uint in concurrent mark
Summary: Change the type of various task num fields, parameters etc to unsigned and rename them to be more consistent with the other collectors. Code changes were also reviewed by Vitaly Davidovich.
Reviewed-by: johnc
Contributed-by: Kaushik Srenevasan <kaushik@twitter.com>
1 /*
2 * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
25 #include "precompiled.hpp"
26 #include "gc_implementation/g1/bufferingOopClosure.hpp"
27 #include "gc_implementation/g1/concurrentG1Refine.hpp"
28 #include "gc_implementation/g1/concurrentG1RefineThread.hpp"
29 #include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp"
30 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
31 #include "gc_implementation/g1/g1CollectorPolicy.hpp"
32 #include "gc_implementation/g1/g1GCPhaseTimes.hpp"
33 #include "gc_implementation/g1/g1OopClosures.inline.hpp"
34 #include "gc_implementation/g1/g1RemSet.inline.hpp"
35 #include "gc_implementation/g1/heapRegionSeq.inline.hpp"
36 #include "memory/iterator.hpp"
37 #include "oops/oop.inline.hpp"
38 #include "utilities/intHisto.hpp"
40 #define CARD_REPEAT_HISTO 0
42 #if CARD_REPEAT_HISTO
43 static size_t ct_freq_sz;
44 static jbyte* ct_freq = NULL;
46 void init_ct_freq_table(size_t heap_sz_bytes) {
47 if (ct_freq == NULL) {
48 ct_freq_sz = heap_sz_bytes/CardTableModRefBS::card_size;
49 ct_freq = new jbyte[ct_freq_sz];
50 for (size_t j = 0; j < ct_freq_sz; j++) ct_freq[j] = 0;
51 }
52 }
54 void ct_freq_note_card(size_t index) {
55 assert(0 <= index && index < ct_freq_sz, "Bounds error.");
56 if (ct_freq[index] < 100) { ct_freq[index]++; }
57 }
59 static IntHistogram card_repeat_count(10, 10);
61 void ct_freq_update_histo_and_reset() {
62 for (size_t j = 0; j < ct_freq_sz; j++) {
63 card_repeat_count.add_entry(ct_freq[j]);
64 ct_freq[j] = 0;
65 }
67 }
68 #endif
70 G1RemSet::G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
71 : _g1(g1), _conc_refine_cards(0),
72 _ct_bs(ct_bs), _g1p(_g1->g1_policy()),
73 _cg1r(g1->concurrent_g1_refine()),
74 _cset_rs_update_cl(NULL),
75 _cards_scanned(NULL), _total_cards_scanned(0)
76 {
77 _seq_task = new SubTasksDone(NumSeqTasks);
78 guarantee(n_workers() > 0, "There should be some workers");
79 _cset_rs_update_cl = NEW_C_HEAP_ARRAY(OopsInHeapRegionClosure*, n_workers(), mtGC);
80 for (uint i = 0; i < n_workers(); i++) {
81 _cset_rs_update_cl[i] = NULL;
82 }
83 }
85 G1RemSet::~G1RemSet() {
86 delete _seq_task;
87 for (uint i = 0; i < n_workers(); i++) {
88 assert(_cset_rs_update_cl[i] == NULL, "it should be");
89 }
90 FREE_C_HEAP_ARRAY(OopsInHeapRegionClosure*, _cset_rs_update_cl, mtGC);
91 }
93 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) {
94 if (_g1->is_in_g1_reserved(mr.start())) {
95 _n += (int) ((mr.byte_size() / CardTableModRefBS::card_size));
96 if (_start_first == NULL) _start_first = mr.start();
97 }
98 }
100 class ScanRSClosure : public HeapRegionClosure {
101 size_t _cards_done, _cards;
102 G1CollectedHeap* _g1h;
103 OopsInHeapRegionClosure* _oc;
104 G1BlockOffsetSharedArray* _bot_shared;
105 CardTableModRefBS *_ct_bs;
106 int _worker_i;
107 int _block_size;
108 bool _try_claimed;
109 public:
110 ScanRSClosure(OopsInHeapRegionClosure* oc, int worker_i) :
111 _oc(oc),
112 _cards(0),
113 _cards_done(0),
114 _worker_i(worker_i),
115 _try_claimed(false)
116 {
117 _g1h = G1CollectedHeap::heap();
118 _bot_shared = _g1h->bot_shared();
119 _ct_bs = (CardTableModRefBS*) (_g1h->barrier_set());
120 _block_size = MAX2<int>(G1RSetScanBlockSize, 1);
121 }
123 void set_try_claimed() { _try_claimed = true; }
125 void scanCard(size_t index, HeapRegion *r) {
126 // Stack allocate the DirtyCardToOopClosure instance
127 HeapRegionDCTOC cl(_g1h, r, _oc,
128 CardTableModRefBS::Precise,
129 HeapRegionDCTOC::IntoCSFilterKind);
131 // Set the "from" region in the closure.
132 _oc->set_region(r);
133 HeapWord* card_start = _bot_shared->address_for_index(index);
134 HeapWord* card_end = card_start + G1BlockOffsetSharedArray::N_words;
135 Space *sp = SharedHeap::heap()->space_containing(card_start);
136 MemRegion sm_region = sp->used_region_at_save_marks();
137 MemRegion mr = sm_region.intersection(MemRegion(card_start,card_end));
138 if (!mr.is_empty() && !_ct_bs->is_card_claimed(index)) {
139 // We make the card as "claimed" lazily (so races are possible
140 // but they're benign), which reduces the number of duplicate
141 // scans (the rsets of the regions in the cset can intersect).
142 _ct_bs->set_card_claimed(index);
143 _cards_done++;
144 cl.do_MemRegion(mr);
145 }
146 }
148 void printCard(HeapRegion* card_region, size_t card_index,
149 HeapWord* card_start) {
150 gclog_or_tty->print_cr("T %d Region [" PTR_FORMAT ", " PTR_FORMAT ") "
151 "RS names card %p: "
152 "[" PTR_FORMAT ", " PTR_FORMAT ")",
153 _worker_i,
154 card_region->bottom(), card_region->end(),
155 card_index,
156 card_start, card_start + G1BlockOffsetSharedArray::N_words);
157 }
159 bool doHeapRegion(HeapRegion* r) {
160 assert(r->in_collection_set(), "should only be called on elements of CS.");
161 HeapRegionRemSet* hrrs = r->rem_set();
162 if (hrrs->iter_is_complete()) return false; // All done.
163 if (!_try_claimed && !hrrs->claim_iter()) return false;
164 // If we ever free the collection set concurrently, we should also
165 // clear the card table concurrently therefore we won't need to
166 // add regions of the collection set to the dirty cards region.
167 _g1h->push_dirty_cards_region(r);
168 // If we didn't return above, then
169 // _try_claimed || r->claim_iter()
170 // is true: either we're supposed to work on claimed-but-not-complete
171 // regions, or we successfully claimed the region.
172 HeapRegionRemSetIterator* iter = _g1h->rem_set_iterator(_worker_i);
173 hrrs->init_iterator(iter);
174 size_t card_index;
176 // We claim cards in block so as to recude the contention. The block size is determined by
177 // the G1RSetScanBlockSize parameter.
178 size_t jump_to_card = hrrs->iter_claimed_next(_block_size);
179 for (size_t current_card = 0; iter->has_next(card_index); current_card++) {
180 if (current_card >= jump_to_card + _block_size) {
181 jump_to_card = hrrs->iter_claimed_next(_block_size);
182 }
183 if (current_card < jump_to_card) continue;
184 HeapWord* card_start = _g1h->bot_shared()->address_for_index(card_index);
185 #if 0
186 gclog_or_tty->print("Rem set iteration yielded card [" PTR_FORMAT ", " PTR_FORMAT ").\n",
187 card_start, card_start + CardTableModRefBS::card_size_in_words);
188 #endif
190 HeapRegion* card_region = _g1h->heap_region_containing(card_start);
191 assert(card_region != NULL, "Yielding cards not in the heap?");
192 _cards++;
194 if (!card_region->is_on_dirty_cards_region_list()) {
195 _g1h->push_dirty_cards_region(card_region);
196 }
198 // If the card is dirty, then we will scan it during updateRS.
199 if (!card_region->in_collection_set() &&
200 !_ct_bs->is_card_dirty(card_index)) {
201 scanCard(card_index, card_region);
202 }
203 }
204 if (!_try_claimed) {
205 hrrs->set_iter_complete();
206 }
207 return false;
208 }
209 size_t cards_done() { return _cards_done;}
210 size_t cards_looked_up() { return _cards;}
211 };
213 void G1RemSet::scanRS(OopsInHeapRegionClosure* oc, int worker_i) {
214 double rs_time_start = os::elapsedTime();
215 HeapRegion *startRegion = _g1->start_cset_region_for_worker(worker_i);
217 ScanRSClosure scanRScl(oc, worker_i);
219 _g1->collection_set_iterate_from(startRegion, &scanRScl);
220 scanRScl.set_try_claimed();
221 _g1->collection_set_iterate_from(startRegion, &scanRScl);
223 double scan_rs_time_sec = os::elapsedTime() - rs_time_start;
225 assert( _cards_scanned != NULL, "invariant" );
226 _cards_scanned[worker_i] = scanRScl.cards_done();
228 _g1p->phase_times()->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0);
229 }
231 // Closure used for updating RSets and recording references that
232 // point into the collection set. Only called during an
233 // evacuation pause.
235 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure {
236 G1RemSet* _g1rs;
237 DirtyCardQueue* _into_cset_dcq;
238 public:
239 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h,
240 DirtyCardQueue* into_cset_dcq) :
241 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq)
242 {}
243 bool do_card_ptr(jbyte* card_ptr, int worker_i) {
244 // The only time we care about recording cards that
245 // contain references that point into the collection set
246 // is during RSet updating within an evacuation pause.
247 // In this case worker_i should be the id of a GC worker thread.
248 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
249 assert(worker_i < (int) (ParallelGCThreads == 0 ? 1 : ParallelGCThreads), "should be a GC worker");
251 if (_g1rs->concurrentRefineOneCard(card_ptr, worker_i, true)) {
252 // 'card_ptr' contains references that point into the collection
253 // set. We need to record the card in the DCQS
254 // (G1CollectedHeap::into_cset_dirty_card_queue_set())
255 // that's used for that purpose.
256 //
257 // Enqueue the card
258 _into_cset_dcq->enqueue(card_ptr);
259 }
260 return true;
261 }
262 };
264 void G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, int worker_i) {
265 double start = os::elapsedTime();
266 // Apply the given closure to all remaining log entries.
267 RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq);
269 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, into_cset_dcq, false, worker_i);
271 // Now there should be no dirty cards.
272 if (G1RSLogCheckCardTable) {
273 CountNonCleanMemRegionClosure cl(_g1);
274 _ct_bs->mod_card_iterate(&cl);
275 // XXX This isn't true any more: keeping cards of young regions
276 // marked dirty broke it. Need some reasonable fix.
277 guarantee(cl.n() == 0, "Card table should be clean.");
278 }
280 _g1p->phase_times()->record_update_rs_time(worker_i, (os::elapsedTime() - start) * 1000.0);
281 }
283 void G1RemSet::cleanupHRRS() {
284 HeapRegionRemSet::cleanup();
285 }
287 void G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
288 int worker_i) {
289 #if CARD_REPEAT_HISTO
290 ct_freq_update_histo_and_reset();
291 #endif
292 if (worker_i == 0) {
293 _cg1r->clear_and_record_card_counts();
294 }
296 // We cache the value of 'oc' closure into the appropriate slot in the
297 // _cset_rs_update_cl for this worker
298 assert(worker_i < (int)n_workers(), "sanity");
299 _cset_rs_update_cl[worker_i] = oc;
301 // A DirtyCardQueue that is used to hold cards containing references
302 // that point into the collection set. This DCQ is associated with a
303 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
304 // circumstances (i.e. the pause successfully completes), these cards
305 // are just discarded (there's no need to update the RSets of regions
306 // that were in the collection set - after the pause these regions
307 // are wholly 'free' of live objects. In the event of an evacuation
308 // failure the cards/buffers in this queue set are:
309 // * passed to the DirtyCardQueueSet that is used to manage deferred
310 // RSet updates, or
311 // * scanned for references that point into the collection set
312 // and the RSet of the corresponding region in the collection set
313 // is updated immediately.
314 DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set());
316 assert((ParallelGCThreads > 0) || worker_i == 0, "invariant");
318 // The two flags below were introduced temporarily to serialize
319 // the updating and scanning of remembered sets. There are some
320 // race conditions when these two operations are done in parallel
321 // and they are causing failures. When we resolve said race
322 // conditions, we'll revert back to parallel remembered set
323 // updating and scanning. See CRs 6677707 and 6677708.
324 if (G1UseParallelRSetUpdating || (worker_i == 0)) {
325 updateRS(&into_cset_dcq, worker_i);
326 } else {
327 _g1p->phase_times()->record_update_rs_processed_buffers(worker_i, 0);
328 _g1p->phase_times()->record_update_rs_time(worker_i, 0.0);
329 }
330 if (G1UseParallelRSetScanning || (worker_i == 0)) {
331 scanRS(oc, worker_i);
332 } else {
333 _g1p->phase_times()->record_scan_rs_time(worker_i, 0.0);
334 }
336 // We now clear the cached values of _cset_rs_update_cl for this worker
337 _cset_rs_update_cl[worker_i] = NULL;
338 }
340 void G1RemSet::prepare_for_oops_into_collection_set_do() {
341 cleanupHRRS();
342 ConcurrentG1Refine* cg1r = _g1->concurrent_g1_refine();
343 _g1->set_refine_cte_cl_concurrency(false);
344 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
345 dcqs.concatenate_logs();
347 if (G1CollectedHeap::use_parallel_gc_threads()) {
348 // Don't set the number of workers here. It will be set
349 // when the task is run
350 // _seq_task->set_n_termination((int)n_workers());
351 }
352 guarantee( _cards_scanned == NULL, "invariant" );
353 _cards_scanned = NEW_C_HEAP_ARRAY(size_t, n_workers(), mtGC);
354 for (uint i = 0; i < n_workers(); ++i) {
355 _cards_scanned[i] = 0;
356 }
357 _total_cards_scanned = 0;
358 }
361 // This closure, applied to a DirtyCardQueueSet, is used to immediately
362 // update the RSets for the regions in the CSet. For each card it iterates
363 // through the oops which coincide with that card. It scans the reference
364 // fields in each oop; when it finds an oop that points into the collection
365 // set, the RSet for the region containing the referenced object is updated.
366 class UpdateRSetCardTableEntryIntoCSetClosure: public CardTableEntryClosure {
367 G1CollectedHeap* _g1;
368 CardTableModRefBS* _ct_bs;
369 public:
370 UpdateRSetCardTableEntryIntoCSetClosure(G1CollectedHeap* g1,
371 CardTableModRefBS* bs):
372 _g1(g1), _ct_bs(bs)
373 { }
375 bool do_card_ptr(jbyte* card_ptr, int worker_i) {
376 // Construct the region representing the card.
377 HeapWord* start = _ct_bs->addr_for(card_ptr);
378 // And find the region containing it.
379 HeapRegion* r = _g1->heap_region_containing(start);
380 assert(r != NULL, "unexpected null");
382 // Scan oops in the card looking for references into the collection set
383 // Don't use addr_for(card_ptr + 1) which can ask for
384 // a card beyond the heap. This is not safe without a perm
385 // gen.
386 HeapWord* end = start + CardTableModRefBS::card_size_in_words;
387 MemRegion scanRegion(start, end);
389 UpdateRSetImmediate update_rs_cl(_g1->g1_rem_set());
390 FilterIntoCSClosure update_rs_cset_oop_cl(NULL, _g1, &update_rs_cl);
391 FilterOutOfRegionClosure filter_then_update_rs_cset_oop_cl(r, &update_rs_cset_oop_cl);
393 // We can pass false as the "filter_young" parameter here as:
394 // * we should be in a STW pause,
395 // * the DCQS to which this closure is applied is used to hold
396 // references that point into the collection set from the prior
397 // RSet updating,
398 // * the post-write barrier shouldn't be logging updates to young
399 // regions (but there is a situation where this can happen - see
400 // the comment in G1RemSet::concurrentRefineOneCard below -
401 // that should not be applicable here), and
402 // * during actual RSet updating, the filtering of cards in young
403 // regions in HeapRegion::oops_on_card_seq_iterate_careful is
404 // employed.
405 // As a result, when this closure is applied to "refs into cset"
406 // DCQS, we shouldn't see any cards in young regions.
407 update_rs_cl.set_region(r);
408 HeapWord* stop_point =
409 r->oops_on_card_seq_iterate_careful(scanRegion,
410 &filter_then_update_rs_cset_oop_cl,
411 false /* filter_young */,
412 NULL /* card_ptr */);
414 // Since this is performed in the event of an evacuation failure, we
415 // we shouldn't see a non-null stop point
416 assert(stop_point == NULL, "saw an unallocated region");
417 return true;
418 }
419 };
421 void G1RemSet::cleanup_after_oops_into_collection_set_do() {
422 guarantee( _cards_scanned != NULL, "invariant" );
423 _total_cards_scanned = 0;
424 for (uint i = 0; i < n_workers(); ++i) {
425 _total_cards_scanned += _cards_scanned[i];
426 }
427 FREE_C_HEAP_ARRAY(size_t, _cards_scanned, mtGC);
428 _cards_scanned = NULL;
429 // Cleanup after copy
430 _g1->set_refine_cte_cl_concurrency(true);
431 // Set all cards back to clean.
432 _g1->cleanUpCardTable();
434 DirtyCardQueueSet& into_cset_dcqs = _g1->into_cset_dirty_card_queue_set();
435 int into_cset_n_buffers = into_cset_dcqs.completed_buffers_num();
437 if (_g1->evacuation_failed()) {
438 // Restore remembered sets for the regions pointing into the collection set.
440 if (G1DeferredRSUpdate) {
441 // If deferred RS updates are enabled then we just need to transfer
442 // the completed buffers from (a) the DirtyCardQueueSet used to hold
443 // cards that contain references that point into the collection set
444 // to (b) the DCQS used to hold the deferred RS updates
445 _g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs);
446 } else {
448 CardTableModRefBS* bs = (CardTableModRefBS*)_g1->barrier_set();
449 UpdateRSetCardTableEntryIntoCSetClosure update_rs_cset_immediate(_g1, bs);
451 int n_completed_buffers = 0;
452 while (into_cset_dcqs.apply_closure_to_completed_buffer(&update_rs_cset_immediate,
453 0, 0, true)) {
454 n_completed_buffers++;
455 }
456 assert(n_completed_buffers == into_cset_n_buffers, "missed some buffers");
457 }
458 }
460 // Free any completed buffers in the DirtyCardQueueSet used to hold cards
461 // which contain references that point into the collection.
462 _g1->into_cset_dirty_card_queue_set().clear();
463 assert(_g1->into_cset_dirty_card_queue_set().completed_buffers_num() == 0,
464 "all buffers should be freed");
465 _g1->into_cset_dirty_card_queue_set().clear_n_completed_buffers();
466 }
468 class ScrubRSClosure: public HeapRegionClosure {
469 G1CollectedHeap* _g1h;
470 BitMap* _region_bm;
471 BitMap* _card_bm;
472 CardTableModRefBS* _ctbs;
473 public:
474 ScrubRSClosure(BitMap* region_bm, BitMap* card_bm) :
475 _g1h(G1CollectedHeap::heap()),
476 _region_bm(region_bm), _card_bm(card_bm),
477 _ctbs(NULL)
478 {
479 ModRefBarrierSet* bs = _g1h->mr_bs();
480 guarantee(bs->is_a(BarrierSet::CardTableModRef), "Precondition");
481 _ctbs = (CardTableModRefBS*)bs;
482 }
484 bool doHeapRegion(HeapRegion* r) {
485 if (!r->continuesHumongous()) {
486 r->rem_set()->scrub(_ctbs, _region_bm, _card_bm);
487 }
488 return false;
489 }
490 };
492 void G1RemSet::scrub(BitMap* region_bm, BitMap* card_bm) {
493 ScrubRSClosure scrub_cl(region_bm, card_bm);
494 _g1->heap_region_iterate(&scrub_cl);
495 }
497 void G1RemSet::scrub_par(BitMap* region_bm, BitMap* card_bm,
498 uint worker_num, int claim_val) {
499 ScrubRSClosure scrub_cl(region_bm, card_bm);
500 _g1->heap_region_par_iterate_chunked(&scrub_cl,
501 worker_num,
502 n_workers(),
503 claim_val);
504 }
508 G1TriggerClosure::G1TriggerClosure() :
509 _triggered(false) { }
511 G1InvokeIfNotTriggeredClosure::G1InvokeIfNotTriggeredClosure(G1TriggerClosure* t_cl,
512 OopClosure* oop_cl) :
513 _trigger_cl(t_cl), _oop_cl(oop_cl) { }
515 G1Mux2Closure::G1Mux2Closure(OopClosure *c1, OopClosure *c2) :
516 _c1(c1), _c2(c2) { }
518 G1UpdateRSOrPushRefOopClosure::
519 G1UpdateRSOrPushRefOopClosure(G1CollectedHeap* g1h,
520 G1RemSet* rs,
521 OopsInHeapRegionClosure* push_ref_cl,
522 bool record_refs_into_cset,
523 int worker_i) :
524 _g1(g1h), _g1_rem_set(rs), _from(NULL),
525 _record_refs_into_cset(record_refs_into_cset),
526 _push_ref_cl(push_ref_cl), _worker_i(worker_i) { }
528 bool G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i,
529 bool check_for_refs_into_cset) {
530 // Construct the region representing the card.
531 HeapWord* start = _ct_bs->addr_for(card_ptr);
532 // And find the region containing it.
533 HeapRegion* r = _g1->heap_region_containing(start);
534 assert(r != NULL, "unexpected null");
536 // Don't use addr_for(card_ptr + 1) which can ask for
537 // a card beyond the heap. This is not safe without a perm
538 // gen at the upper end of the heap.
539 HeapWord* end = start + CardTableModRefBS::card_size_in_words;
540 MemRegion dirtyRegion(start, end);
542 #if CARD_REPEAT_HISTO
543 init_ct_freq_table(_g1->max_capacity());
544 ct_freq_note_card(_ct_bs->index_for(start));
545 #endif
547 OopsInHeapRegionClosure* oops_in_heap_closure = NULL;
548 if (check_for_refs_into_cset) {
549 // ConcurrentG1RefineThreads have worker numbers larger than what
550 // _cset_rs_update_cl[] is set up to handle. But those threads should
551 // only be active outside of a collection which means that when they
552 // reach here they should have check_for_refs_into_cset == false.
553 assert((size_t)worker_i < n_workers(), "index of worker larger than _cset_rs_update_cl[].length");
554 oops_in_heap_closure = _cset_rs_update_cl[worker_i];
555 }
556 G1UpdateRSOrPushRefOopClosure update_rs_oop_cl(_g1,
557 _g1->g1_rem_set(),
558 oops_in_heap_closure,
559 check_for_refs_into_cset,
560 worker_i);
561 update_rs_oop_cl.set_from(r);
563 G1TriggerClosure trigger_cl;
564 FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl);
565 G1InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl);
566 G1Mux2Closure mux(&invoke_cl, &update_rs_oop_cl);
568 FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r,
569 (check_for_refs_into_cset ?
570 (OopClosure*)&mux :
571 (OopClosure*)&update_rs_oop_cl));
573 // The region for the current card may be a young region. The
574 // current card may have been a card that was evicted from the
575 // card cache. When the card was inserted into the cache, we had
576 // determined that its region was non-young. While in the cache,
577 // the region may have been freed during a cleanup pause, reallocated
578 // and tagged as young.
579 //
580 // We wish to filter out cards for such a region but the current
581 // thread, if we're running concurrently, may "see" the young type
582 // change at any time (so an earlier "is_young" check may pass or
583 // fail arbitrarily). We tell the iteration code to perform this
584 // filtering when it has been determined that there has been an actual
585 // allocation in this region and making it safe to check the young type.
586 bool filter_young = true;
588 HeapWord* stop_point =
589 r->oops_on_card_seq_iterate_careful(dirtyRegion,
590 &filter_then_update_rs_oop_cl,
591 filter_young,
592 card_ptr);
594 // If stop_point is non-null, then we encountered an unallocated region
595 // (perhaps the unfilled portion of a TLAB.) For now, we'll dirty the
596 // card and re-enqueue: if we put off the card until a GC pause, then the
597 // unallocated portion will be filled in. Alternatively, we might try
598 // the full complexity of the technique used in "regular" precleaning.
599 if (stop_point != NULL) {
600 // The card might have gotten re-dirtied and re-enqueued while we
601 // worked. (In fact, it's pretty likely.)
602 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
603 *card_ptr = CardTableModRefBS::dirty_card_val();
604 MutexLockerEx x(Shared_DirtyCardQ_lock,
605 Mutex::_no_safepoint_check_flag);
606 DirtyCardQueue* sdcq =
607 JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
608 sdcq->enqueue(card_ptr);
609 }
610 } else {
611 _conc_refine_cards++;
612 }
614 return trigger_cl.triggered();
615 }
617 bool G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i,
618 bool check_for_refs_into_cset) {
619 // If the card is no longer dirty, nothing to do.
620 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
621 // No need to return that this card contains refs that point
622 // into the collection set.
623 return false;
624 }
626 // Construct the region representing the card.
627 HeapWord* start = _ct_bs->addr_for(card_ptr);
628 // And find the region containing it.
629 HeapRegion* r = _g1->heap_region_containing(start);
630 if (r == NULL) {
631 // Again no need to return that this card contains refs that
632 // point into the collection set.
633 return false; // Not in the G1 heap (might be in perm, for example.)
634 }
635 // Why do we have to check here whether a card is on a young region,
636 // given that we dirty young regions and, as a result, the
637 // post-barrier is supposed to filter them out and never to enqueue
638 // them? When we allocate a new region as the "allocation region" we
639 // actually dirty its cards after we release the lock, since card
640 // dirtying while holding the lock was a performance bottleneck. So,
641 // as a result, it is possible for other threads to actually
642 // allocate objects in the region (after the acquire the lock)
643 // before all the cards on the region are dirtied. This is unlikely,
644 // and it doesn't happen often, but it can happen. So, the extra
645 // check below filters out those cards.
646 if (r->is_young()) {
647 return false;
648 }
649 // While we are processing RSet buffers during the collection, we
650 // actually don't want to scan any cards on the collection set,
651 // since we don't want to update remebered sets with entries that
652 // point into the collection set, given that live objects from the
653 // collection set are about to move and such entries will be stale
654 // very soon. This change also deals with a reliability issue which
655 // involves scanning a card in the collection set and coming across
656 // an array that was being chunked and looking malformed. Note,
657 // however, that if evacuation fails, we have to scan any objects
658 // that were not moved and create any missing entries.
659 if (r->in_collection_set()) {
660 return false;
661 }
663 // Should we defer processing the card?
664 //
665 // Previously the result from the insert_cache call would be
666 // either card_ptr (implying that card_ptr was currently "cold"),
667 // null (meaning we had inserted the card ptr into the "hot"
668 // cache, which had some headroom), or a "hot" card ptr
669 // extracted from the "hot" cache.
670 //
671 // Now that the _card_counts cache in the ConcurrentG1Refine
672 // instance is an evicting hash table, the result we get back
673 // could be from evicting the card ptr in an already occupied
674 // bucket (in which case we have replaced the card ptr in the
675 // bucket with card_ptr and "defer" is set to false). To avoid
676 // having a data structure (updates to which would need a lock)
677 // to hold these unprocessed dirty cards, we need to immediately
678 // process card_ptr. The actions needed to be taken on return
679 // from cache_insert are summarized in the following table:
680 //
681 // res defer action
682 // --------------------------------------------------------------
683 // null false card evicted from _card_counts & replaced with
684 // card_ptr; evicted ptr added to hot cache.
685 // No need to process res; immediately process card_ptr
686 //
687 // null true card not evicted from _card_counts; card_ptr added
688 // to hot cache.
689 // Nothing to do.
690 //
691 // non-null false card evicted from _card_counts & replaced with
692 // card_ptr; evicted ptr is currently "cold" or
693 // caused an eviction from the hot cache.
694 // Immediately process res; process card_ptr.
695 //
696 // non-null true card not evicted from _card_counts; card_ptr is
697 // currently cold, or caused an eviction from hot
698 // cache.
699 // Immediately process res; no need to process card_ptr.
702 jbyte* res = card_ptr;
703 bool defer = false;
705 // This gets set to true if the card being refined has references
706 // that point into the collection set.
707 bool oops_into_cset = false;
709 if (_cg1r->use_cache()) {
710 jbyte* res = _cg1r->cache_insert(card_ptr, &defer);
711 if (res != NULL && (res != card_ptr || defer)) {
712 start = _ct_bs->addr_for(res);
713 r = _g1->heap_region_containing(start);
714 if (r != NULL) {
715 // Checking whether the region we got back from the cache
716 // is young here is inappropriate. The region could have been
717 // freed, reallocated and tagged as young while in the cache.
718 // Hence we could see its young type change at any time.
719 //
720 // Process card pointer we get back from the hot card cache. This
721 // will check whether the region containing the card is young
722 // _after_ checking that the region has been allocated from.
723 oops_into_cset = concurrentRefineOneCard_impl(res, worker_i,
724 false /* check_for_refs_into_cset */);
725 // The above call to concurrentRefineOneCard_impl is only
726 // performed if the hot card cache is enabled. This cache is
727 // disabled during an evacuation pause - which is the only
728 // time when we need know if the card contains references
729 // that point into the collection set. Also when the hot card
730 // cache is enabled, this code is executed by the concurrent
731 // refine threads - rather than the GC worker threads - and
732 // concurrentRefineOneCard_impl will return false.
733 assert(!oops_into_cset, "should not see true here");
734 }
735 }
736 }
738 if (!defer) {
739 oops_into_cset =
740 concurrentRefineOneCard_impl(card_ptr, worker_i, check_for_refs_into_cset);
741 // We should only be detecting that the card contains references
742 // that point into the collection set if the current thread is
743 // a GC worker thread.
744 assert(!oops_into_cset || SafepointSynchronize::is_at_safepoint(),
745 "invalid result at non safepoint");
746 }
747 return oops_into_cset;
748 }
750 class HRRSStatsIter: public HeapRegionClosure {
751 size_t _occupied;
752 size_t _total_mem_sz;
753 size_t _max_mem_sz;
754 HeapRegion* _max_mem_sz_region;
755 public:
756 HRRSStatsIter() :
757 _occupied(0),
758 _total_mem_sz(0),
759 _max_mem_sz(0),
760 _max_mem_sz_region(NULL)
761 {}
763 bool doHeapRegion(HeapRegion* r) {
764 if (r->continuesHumongous()) return false;
765 size_t mem_sz = r->rem_set()->mem_size();
766 if (mem_sz > _max_mem_sz) {
767 _max_mem_sz = mem_sz;
768 _max_mem_sz_region = r;
769 }
770 _total_mem_sz += mem_sz;
771 size_t occ = r->rem_set()->occupied();
772 _occupied += occ;
773 return false;
774 }
775 size_t total_mem_sz() { return _total_mem_sz; }
776 size_t max_mem_sz() { return _max_mem_sz; }
777 size_t occupied() { return _occupied; }
778 HeapRegion* max_mem_sz_region() { return _max_mem_sz_region; }
779 };
781 class PrintRSThreadVTimeClosure : public ThreadClosure {
782 public:
783 virtual void do_thread(Thread *t) {
784 ConcurrentG1RefineThread* crt = (ConcurrentG1RefineThread*) t;
785 gclog_or_tty->print(" %5.2f", crt->vtime_accum());
786 }
787 };
789 void G1RemSet::print_summary_info() {
790 G1CollectedHeap* g1 = G1CollectedHeap::heap();
792 #if CARD_REPEAT_HISTO
793 gclog_or_tty->print_cr("\nG1 card_repeat count histogram: ");
794 gclog_or_tty->print_cr(" # of repeats --> # of cards with that number.");
795 card_repeat_count.print_on(gclog_or_tty);
796 #endif
798 gclog_or_tty->print_cr("\n Concurrent RS processed %d cards",
799 _conc_refine_cards);
800 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
801 jint tot_processed_buffers =
802 dcqs.processed_buffers_mut() + dcqs.processed_buffers_rs_thread();
803 gclog_or_tty->print_cr(" Of %d completed buffers:", tot_processed_buffers);
804 gclog_or_tty->print_cr(" %8d (%5.1f%%) by conc RS threads.",
805 dcqs.processed_buffers_rs_thread(),
806 100.0*(float)dcqs.processed_buffers_rs_thread()/
807 (float)tot_processed_buffers);
808 gclog_or_tty->print_cr(" %8d (%5.1f%%) by mutator threads.",
809 dcqs.processed_buffers_mut(),
810 100.0*(float)dcqs.processed_buffers_mut()/
811 (float)tot_processed_buffers);
812 gclog_or_tty->print_cr(" Conc RS threads times(s)");
813 PrintRSThreadVTimeClosure p;
814 gclog_or_tty->print(" ");
815 g1->concurrent_g1_refine()->threads_do(&p);
816 gclog_or_tty->print_cr("");
818 HRRSStatsIter blk;
819 g1->heap_region_iterate(&blk);
820 gclog_or_tty->print_cr(" Total heap region rem set sizes = "SIZE_FORMAT"K."
821 " Max = "SIZE_FORMAT"K.",
822 blk.total_mem_sz()/K, blk.max_mem_sz()/K);
823 gclog_or_tty->print_cr(" Static structures = "SIZE_FORMAT"K,"
824 " free_lists = "SIZE_FORMAT"K.",
825 HeapRegionRemSet::static_mem_size() / K,
826 HeapRegionRemSet::fl_mem_size() / K);
827 gclog_or_tty->print_cr(" "SIZE_FORMAT" occupied cards represented.",
828 blk.occupied());
829 HeapRegion* max_mem_sz_region = blk.max_mem_sz_region();
830 HeapRegionRemSet* rem_set = max_mem_sz_region->rem_set();
831 gclog_or_tty->print_cr(" Max size region = "HR_FORMAT", "
832 "size = "SIZE_FORMAT "K, occupied = "SIZE_FORMAT"K.",
833 HR_FORMAT_PARAMS(max_mem_sz_region),
834 (rem_set->mem_size() + K - 1)/K,
835 (rem_set->occupied() + K - 1)/K);
836 gclog_or_tty->print_cr(" Did %d coarsenings.",
837 HeapRegionRemSet::n_coarsenings());
838 }
840 void G1RemSet::prepare_for_verify() {
841 if (G1HRRSFlushLogBuffersOnVerify &&
842 (VerifyBeforeGC || VerifyAfterGC)
843 && !_g1->full_collection()) {
844 cleanupHRRS();
845 _g1->set_refine_cte_cl_concurrency(false);
846 if (SafepointSynchronize::is_at_safepoint()) {
847 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
848 dcqs.concatenate_logs();
849 }
850 bool cg1r_use_cache = _cg1r->use_cache();
851 _cg1r->set_use_cache(false);
852 DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set());
853 updateRS(&into_cset_dcq, 0);
854 _g1->into_cset_dirty_card_queue_set().clear();
855 _cg1r->set_use_cache(cg1r_use_cache);
857 assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed");
858 }
859 }