Sat, 07 Mar 2009 11:07:37 -0500
6814467: G1: small fixes related to concurrent marking verboseness
Summary: A few small fixes to remove some inconsistencies in the concurrent mark-related verbose GC output.
Reviewed-by: jmasa
1 /*
2 * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
20 * CA 95054 USA or visit www.sun.com if you need additional information or
21 * have any questions.
22 *
23 */
25 #include "incls/_precompiled.incl"
26 #include "incls/_g1RemSet.cpp.incl"
28 #define CARD_REPEAT_HISTO 0
30 #if CARD_REPEAT_HISTO
31 static size_t ct_freq_sz;
32 static jbyte* ct_freq = NULL;
34 void init_ct_freq_table(size_t heap_sz_bytes) {
35 if (ct_freq == NULL) {
36 ct_freq_sz = heap_sz_bytes/CardTableModRefBS::card_size;
37 ct_freq = new jbyte[ct_freq_sz];
38 for (size_t j = 0; j < ct_freq_sz; j++) ct_freq[j] = 0;
39 }
40 }
42 void ct_freq_note_card(size_t index) {
43 assert(0 <= index && index < ct_freq_sz, "Bounds error.");
44 if (ct_freq[index] < 100) { ct_freq[index]++; }
45 }
47 static IntHistogram card_repeat_count(10, 10);
49 void ct_freq_update_histo_and_reset() {
50 for (size_t j = 0; j < ct_freq_sz; j++) {
51 card_repeat_count.add_entry(ct_freq[j]);
52 ct_freq[j] = 0;
53 }
55 }
56 #endif
59 class IntoCSOopClosure: public OopsInHeapRegionClosure {
60 OopsInHeapRegionClosure* _blk;
61 G1CollectedHeap* _g1;
62 public:
63 IntoCSOopClosure(G1CollectedHeap* g1, OopsInHeapRegionClosure* blk) :
64 _g1(g1), _blk(blk) {}
65 void set_region(HeapRegion* from) {
66 _blk->set_region(from);
67 }
68 virtual void do_oop(narrowOop* p) {
69 guarantee(false, "NYI");
70 }
71 virtual void do_oop(oop* p) {
72 oop obj = *p;
73 if (_g1->obj_in_cs(obj)) _blk->do_oop(p);
74 }
75 bool apply_to_weak_ref_discovered_field() { return true; }
76 bool idempotent() { return true; }
77 };
79 class IntoCSRegionClosure: public HeapRegionClosure {
80 IntoCSOopClosure _blk;
81 G1CollectedHeap* _g1;
82 public:
83 IntoCSRegionClosure(G1CollectedHeap* g1, OopsInHeapRegionClosure* blk) :
84 _g1(g1), _blk(g1, blk) {}
85 bool doHeapRegion(HeapRegion* r) {
86 if (!r->in_collection_set()) {
87 _blk.set_region(r);
88 if (r->isHumongous()) {
89 if (r->startsHumongous()) {
90 oop obj = oop(r->bottom());
91 obj->oop_iterate(&_blk);
92 }
93 } else {
94 r->oop_before_save_marks_iterate(&_blk);
95 }
96 }
97 return false;
98 }
99 };
101 void
102 StupidG1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
103 int worker_i) {
104 IntoCSRegionClosure rc(_g1, oc);
105 _g1->heap_region_iterate(&rc);
106 }
108 class UpdateRSOopClosure: public OopClosure {
109 HeapRegion* _from;
110 HRInto_G1RemSet* _rs;
111 int _worker_i;
112 public:
113 UpdateRSOopClosure(HRInto_G1RemSet* rs, int worker_i = 0) :
114 _from(NULL), _rs(rs), _worker_i(worker_i) {
115 guarantee(_rs != NULL, "Requires an HRIntoG1RemSet");
116 }
118 void set_from(HeapRegion* from) {
119 assert(from != NULL, "from region must be non-NULL");
120 _from = from;
121 }
123 virtual void do_oop(narrowOop* p) {
124 guarantee(false, "NYI");
125 }
126 virtual void do_oop(oop* p) {
127 assert(_from != NULL, "from region must be non-NULL");
128 _rs->par_write_ref(_from, p, _worker_i);
129 }
130 // Override: this closure is idempotent.
131 // bool idempotent() { return true; }
132 bool apply_to_weak_ref_discovered_field() { return true; }
133 };
135 class UpdateRSOutOfRegionClosure: public HeapRegionClosure {
136 G1CollectedHeap* _g1h;
137 ModRefBarrierSet* _mr_bs;
138 UpdateRSOopClosure _cl;
139 int _worker_i;
140 public:
141 UpdateRSOutOfRegionClosure(G1CollectedHeap* g1, int worker_i = 0) :
142 _cl(g1->g1_rem_set()->as_HRInto_G1RemSet(), worker_i),
143 _mr_bs(g1->mr_bs()),
144 _worker_i(worker_i),
145 _g1h(g1)
146 {}
147 bool doHeapRegion(HeapRegion* r) {
148 if (!r->in_collection_set() && !r->continuesHumongous()) {
149 _cl.set_from(r);
150 r->set_next_filter_kind(HeapRegionDCTOC::OutOfRegionFilterKind);
151 _mr_bs->mod_oop_in_space_iterate(r, &_cl, true, true);
152 }
153 return false;
154 }
155 };
157 class VerifyRSCleanCardOopClosure: public OopClosure {
158 G1CollectedHeap* _g1;
159 public:
160 VerifyRSCleanCardOopClosure(G1CollectedHeap* g1) : _g1(g1) {}
162 virtual void do_oop(narrowOop* p) {
163 guarantee(false, "NYI");
164 }
165 virtual void do_oop(oop* p) {
166 oop obj = *p;
167 HeapRegion* to = _g1->heap_region_containing(obj);
168 guarantee(to == NULL || !to->in_collection_set(),
169 "Missed a rem set member.");
170 }
171 };
173 HRInto_G1RemSet::HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
174 : G1RemSet(g1), _ct_bs(ct_bs), _g1p(_g1->g1_policy()),
175 _cg1r(g1->concurrent_g1_refine()),
176 _par_traversal_in_progress(false), _new_refs(NULL),
177 _cards_scanned(NULL), _total_cards_scanned(0)
178 {
179 _seq_task = new SubTasksDone(NumSeqTasks);
180 guarantee(n_workers() > 0, "There should be some workers");
181 _new_refs = NEW_C_HEAP_ARRAY(GrowableArray<oop*>*, n_workers());
182 for (uint i = 0; i < n_workers(); i++) {
183 _new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<oop*>(8192,true);
184 }
185 }
187 HRInto_G1RemSet::~HRInto_G1RemSet() {
188 delete _seq_task;
189 for (uint i = 0; i < n_workers(); i++) {
190 delete _new_refs[i];
191 }
192 FREE_C_HEAP_ARRAY(GrowableArray<oop*>*, _new_refs);
193 }
195 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) {
196 if (_g1->is_in_g1_reserved(mr.start())) {
197 _n += (int) ((mr.byte_size() / CardTableModRefBS::card_size));
198 if (_start_first == NULL) _start_first = mr.start();
199 }
200 }
202 class ScanRSClosure : public HeapRegionClosure {
203 size_t _cards_done, _cards;
204 G1CollectedHeap* _g1h;
205 OopsInHeapRegionClosure* _oc;
206 G1BlockOffsetSharedArray* _bot_shared;
207 CardTableModRefBS *_ct_bs;
208 int _worker_i;
209 bool _try_claimed;
210 public:
211 ScanRSClosure(OopsInHeapRegionClosure* oc, int worker_i) :
212 _oc(oc),
213 _cards(0),
214 _cards_done(0),
215 _worker_i(worker_i),
216 _try_claimed(false)
217 {
218 _g1h = G1CollectedHeap::heap();
219 _bot_shared = _g1h->bot_shared();
220 _ct_bs = (CardTableModRefBS*) (_g1h->barrier_set());
221 }
223 void set_try_claimed() { _try_claimed = true; }
225 void scanCard(size_t index, HeapRegion *r) {
226 _cards_done++;
227 DirtyCardToOopClosure* cl =
228 r->new_dcto_closure(_oc,
229 CardTableModRefBS::Precise,
230 HeapRegionDCTOC::IntoCSFilterKind);
232 // Set the "from" region in the closure.
233 _oc->set_region(r);
234 HeapWord* card_start = _bot_shared->address_for_index(index);
235 HeapWord* card_end = card_start + G1BlockOffsetSharedArray::N_words;
236 Space *sp = SharedHeap::heap()->space_containing(card_start);
237 MemRegion sm_region;
238 if (ParallelGCThreads > 0) {
239 // first find the used area
240 sm_region = sp->used_region_at_save_marks();
241 } else {
242 // The closure is not idempotent. We shouldn't look at objects
243 // allocated during the GC.
244 sm_region = sp->used_region_at_save_marks();
245 }
246 MemRegion mr = sm_region.intersection(MemRegion(card_start,card_end));
247 if (!mr.is_empty()) {
248 cl->do_MemRegion(mr);
249 }
250 }
252 void printCard(HeapRegion* card_region, size_t card_index,
253 HeapWord* card_start) {
254 gclog_or_tty->print_cr("T %d Region [" PTR_FORMAT ", " PTR_FORMAT ") "
255 "RS names card %p: "
256 "[" PTR_FORMAT ", " PTR_FORMAT ")",
257 _worker_i,
258 card_region->bottom(), card_region->end(),
259 card_index,
260 card_start, card_start + G1BlockOffsetSharedArray::N_words);
261 }
263 bool doHeapRegion(HeapRegion* r) {
264 assert(r->in_collection_set(), "should only be called on elements of CS.");
265 HeapRegionRemSet* hrrs = r->rem_set();
266 if (hrrs->iter_is_complete()) return false; // All done.
267 if (!_try_claimed && !hrrs->claim_iter()) return false;
268 // If we didn't return above, then
269 // _try_claimed || r->claim_iter()
270 // is true: either we're supposed to work on claimed-but-not-complete
271 // regions, or we successfully claimed the region.
272 HeapRegionRemSetIterator* iter = _g1h->rem_set_iterator(_worker_i);
273 hrrs->init_iterator(iter);
274 size_t card_index;
275 while (iter->has_next(card_index)) {
276 HeapWord* card_start = _g1h->bot_shared()->address_for_index(card_index);
278 #if 0
279 gclog_or_tty->print("Rem set iteration yielded card [" PTR_FORMAT ", " PTR_FORMAT ").\n",
280 card_start, card_start + CardTableModRefBS::card_size_in_words);
281 #endif
283 HeapRegion* card_region = _g1h->heap_region_containing(card_start);
284 assert(card_region != NULL, "Yielding cards not in the heap?");
285 _cards++;
287 if (!card_region->in_collection_set()) {
288 // If the card is dirty, then we will scan it during updateRS.
289 if (!_ct_bs->is_card_claimed(card_index) &&
290 !_ct_bs->is_card_dirty(card_index)) {
291 assert(_ct_bs->is_card_clean(card_index) ||
292 _ct_bs->is_card_claimed(card_index) ||
293 _ct_bs->is_card_deferred(card_index),
294 "Card is either clean, claimed or deferred");
295 if (_ct_bs->claim_card(card_index))
296 scanCard(card_index, card_region);
297 }
298 }
299 }
300 hrrs->set_iter_complete();
301 return false;
302 }
303 // Set all cards back to clean.
304 void cleanup() {_g1h->cleanUpCardTable();}
305 size_t cards_done() { return _cards_done;}
306 size_t cards_looked_up() { return _cards;}
307 };
309 // We want the parallel threads to start their scanning at
310 // different collection set regions to avoid contention.
311 // If we have:
312 // n collection set regions
313 // p threads
314 // Then thread t will start at region t * floor (n/p)
316 HeapRegion* HRInto_G1RemSet::calculateStartRegion(int worker_i) {
317 HeapRegion* result = _g1p->collection_set();
318 if (ParallelGCThreads > 0) {
319 size_t cs_size = _g1p->collection_set_size();
320 int n_workers = _g1->workers()->total_workers();
321 size_t cs_spans = cs_size / n_workers;
322 size_t ind = cs_spans * worker_i;
323 for (size_t i = 0; i < ind; i++)
324 result = result->next_in_collection_set();
325 }
326 return result;
327 }
329 void HRInto_G1RemSet::scanRS(OopsInHeapRegionClosure* oc, int worker_i) {
330 double rs_time_start = os::elapsedTime();
331 HeapRegion *startRegion = calculateStartRegion(worker_i);
333 BufferingOopsInHeapRegionClosure boc(oc);
334 ScanRSClosure scanRScl(&boc, worker_i);
335 _g1->collection_set_iterate_from(startRegion, &scanRScl);
336 scanRScl.set_try_claimed();
337 _g1->collection_set_iterate_from(startRegion, &scanRScl);
339 boc.done();
340 double closure_app_time_sec = boc.closure_app_seconds();
341 double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) -
342 closure_app_time_sec;
343 double closure_app_time_ms = closure_app_time_sec * 1000.0;
345 assert( _cards_scanned != NULL, "invariant" );
346 _cards_scanned[worker_i] = scanRScl.cards_done();
348 _g1p->record_scan_rs_start_time(worker_i, rs_time_start * 1000.0);
349 _g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0);
351 double scan_new_refs_time_ms = _g1p->get_scan_new_refs_time(worker_i);
352 if (scan_new_refs_time_ms > 0.0) {
353 closure_app_time_ms += scan_new_refs_time_ms;
354 }
356 _g1p->record_obj_copy_time(worker_i, closure_app_time_ms);
357 }
359 void HRInto_G1RemSet::updateRS(int worker_i) {
360 ConcurrentG1Refine* cg1r = _g1->concurrent_g1_refine();
362 double start = os::elapsedTime();
363 _g1p->record_update_rs_start_time(worker_i, start * 1000.0);
365 if (G1RSBarrierUseQueue && !cg1r->do_traversal()) {
366 // Apply the appropriate closure to all remaining log entries.
367 _g1->iterate_dirty_card_closure(false, worker_i);
368 // Now there should be no dirty cards.
369 if (G1RSLogCheckCardTable) {
370 CountNonCleanMemRegionClosure cl(_g1);
371 _ct_bs->mod_card_iterate(&cl);
372 // XXX This isn't true any more: keeping cards of young regions
373 // marked dirty broke it. Need some reasonable fix.
374 guarantee(cl.n() == 0, "Card table should be clean.");
375 }
376 } else {
377 UpdateRSOutOfRegionClosure update_rs(_g1, worker_i);
378 _g1->heap_region_iterate(&update_rs);
379 // We did a traversal; no further one is necessary.
380 if (G1RSBarrierUseQueue) {
381 assert(cg1r->do_traversal(), "Or we shouldn't have gotten here.");
382 cg1r->set_pya_cancel();
383 }
384 if (_cg1r->use_cache()) {
385 _cg1r->clear_and_record_card_counts();
386 _cg1r->clear_hot_cache();
387 }
388 }
389 _g1p->record_update_rs_time(worker_i, (os::elapsedTime() - start) * 1000.0);
390 }
392 #ifndef PRODUCT
393 class PrintRSClosure : public HeapRegionClosure {
394 int _count;
395 public:
396 PrintRSClosure() : _count(0) {}
397 bool doHeapRegion(HeapRegion* r) {
398 HeapRegionRemSet* hrrs = r->rem_set();
399 _count += (int) hrrs->occupied();
400 if (hrrs->occupied() == 0) {
401 gclog_or_tty->print("Heap Region [" PTR_FORMAT ", " PTR_FORMAT ") "
402 "has no remset entries\n",
403 r->bottom(), r->end());
404 } else {
405 gclog_or_tty->print("Printing rem set for heap region [" PTR_FORMAT ", " PTR_FORMAT ")\n",
406 r->bottom(), r->end());
407 r->print();
408 hrrs->print();
409 gclog_or_tty->print("\nDone printing rem set\n");
410 }
411 return false;
412 }
413 int occupied() {return _count;}
414 };
415 #endif
417 class CountRSSizeClosure: public HeapRegionClosure {
418 size_t _n;
419 size_t _tot;
420 size_t _max;
421 HeapRegion* _max_r;
422 enum {
423 N = 20,
424 MIN = 6
425 };
426 int _histo[N];
427 public:
428 CountRSSizeClosure() : _n(0), _tot(0), _max(0), _max_r(NULL) {
429 for (int i = 0; i < N; i++) _histo[i] = 0;
430 }
431 bool doHeapRegion(HeapRegion* r) {
432 if (!r->continuesHumongous()) {
433 size_t occ = r->rem_set()->occupied();
434 _n++;
435 _tot += occ;
436 if (occ > _max) {
437 _max = occ;
438 _max_r = r;
439 }
440 // Fit it into a histo bin.
441 int s = 1 << MIN;
442 int i = 0;
443 while (occ > (size_t) s && i < (N-1)) {
444 s = s << 1;
445 i++;
446 }
447 _histo[i]++;
448 }
449 return false;
450 }
451 size_t n() { return _n; }
452 size_t tot() { return _tot; }
453 size_t mx() { return _max; }
454 HeapRegion* mxr() { return _max_r; }
455 void print_histo() {
456 int mx = N;
457 while (mx >= 0) {
458 if (_histo[mx-1] > 0) break;
459 mx--;
460 }
461 gclog_or_tty->print_cr("Number of regions with given RS sizes:");
462 gclog_or_tty->print_cr(" <= %8d %8d", 1 << MIN, _histo[0]);
463 for (int i = 1; i < mx-1; i++) {
464 gclog_or_tty->print_cr(" %8d - %8d %8d",
465 (1 << (MIN + i - 1)) + 1,
466 1 << (MIN + i),
467 _histo[i]);
468 }
469 gclog_or_tty->print_cr(" > %8d %8d", (1 << (MIN+mx-2))+1, _histo[mx-1]);
470 }
471 };
473 void
474 HRInto_G1RemSet::scanNewRefsRS(OopsInHeapRegionClosure* oc,
475 int worker_i) {
476 double scan_new_refs_start_sec = os::elapsedTime();
477 G1CollectedHeap* g1h = G1CollectedHeap::heap();
478 CardTableModRefBS* ct_bs = (CardTableModRefBS*) (g1h->barrier_set());
479 for (int i = 0; i < _new_refs[worker_i]->length(); i++) {
480 oop* p = _new_refs[worker_i]->at(i);
481 oop obj = *p;
482 // *p was in the collection set when p was pushed on "_new_refs", but
483 // another thread may have processed this location from an RS, so it
484 // might not point into the CS any longer. If so, it's obviously been
485 // processed, and we don't need to do anything further.
486 if (g1h->obj_in_cs(obj)) {
487 HeapRegion* r = g1h->heap_region_containing(p);
489 DEBUG_ONLY(HeapRegion* to = g1h->heap_region_containing(obj));
490 oc->set_region(r);
491 // If "p" has already been processed concurrently, this is
492 // idempotent.
493 oc->do_oop(p);
494 }
495 }
496 _g1p->record_scan_new_refs_time(worker_i,
497 (os::elapsedTime() - scan_new_refs_start_sec)
498 * 1000.0);
499 }
501 void HRInto_G1RemSet::set_par_traversal(bool b) {
502 _par_traversal_in_progress = b;
503 HeapRegionRemSet::set_par_traversal(b);
504 }
506 void HRInto_G1RemSet::cleanupHRRS() {
507 HeapRegionRemSet::cleanup();
508 }
510 void
511 HRInto_G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
512 int worker_i) {
513 #if CARD_REPEAT_HISTO
514 ct_freq_update_histo_and_reset();
515 #endif
516 if (worker_i == 0) {
517 _cg1r->clear_and_record_card_counts();
518 }
520 // Make this into a command-line flag...
521 if (G1RSCountHisto && (ParallelGCThreads == 0 || worker_i == 0)) {
522 CountRSSizeClosure count_cl;
523 _g1->heap_region_iterate(&count_cl);
524 gclog_or_tty->print_cr("Avg of %d RS counts is %f, max is %d, "
525 "max region is " PTR_FORMAT,
526 count_cl.n(), (float)count_cl.tot()/(float)count_cl.n(),
527 count_cl.mx(), count_cl.mxr());
528 count_cl.print_histo();
529 }
531 if (ParallelGCThreads > 0) {
532 // This is a temporary change to serialize the update and scanning
533 // of remembered sets. There are some race conditions when this is
534 // done in parallel and they are causing failures. When we resolve
535 // said race conditions, we'll revert back to parallel remembered
536 // set updating and scanning. See CRs 6677707 and 6677708.
537 if (worker_i == 0) {
538 updateRS(worker_i);
539 scanNewRefsRS(oc, worker_i);
540 scanRS(oc, worker_i);
541 }
542 } else {
543 assert(worker_i == 0, "invariant");
544 updateRS(0);
545 scanNewRefsRS(oc, 0);
546 scanRS(oc, 0);
547 }
548 }
550 void HRInto_G1RemSet::
551 prepare_for_oops_into_collection_set_do() {
552 #if G1_REM_SET_LOGGING
553 PrintRSClosure cl;
554 _g1->collection_set_iterate(&cl);
555 #endif
556 cleanupHRRS();
557 ConcurrentG1Refine* cg1r = _g1->concurrent_g1_refine();
558 _g1->set_refine_cte_cl_concurrency(false);
559 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
560 dcqs.concatenate_logs();
562 assert(!_par_traversal_in_progress, "Invariant between iterations.");
563 if (ParallelGCThreads > 0) {
564 set_par_traversal(true);
565 _seq_task->set_par_threads((int)n_workers());
566 if (cg1r->do_traversal()) {
567 updateRS(0);
568 // Have to do this again after updaters
569 cleanupHRRS();
570 }
571 }
572 guarantee( _cards_scanned == NULL, "invariant" );
573 _cards_scanned = NEW_C_HEAP_ARRAY(size_t, n_workers());
574 for (uint i = 0; i < n_workers(); ++i) {
575 _cards_scanned[i] = 0;
576 }
577 _total_cards_scanned = 0;
578 }
581 class cleanUpIteratorsClosure : public HeapRegionClosure {
582 bool doHeapRegion(HeapRegion *r) {
583 HeapRegionRemSet* hrrs = r->rem_set();
584 hrrs->init_for_par_iteration();
585 return false;
586 }
587 };
589 class UpdateRSetOopsIntoCSImmediate : public OopClosure {
590 G1CollectedHeap* _g1;
591 public:
592 UpdateRSetOopsIntoCSImmediate(G1CollectedHeap* g1) : _g1(g1) { }
593 virtual void do_oop(narrowOop* p) {
594 guarantee(false, "NYI");
595 }
596 virtual void do_oop(oop* p) {
597 HeapRegion* to = _g1->heap_region_containing(*p);
598 if (to->in_collection_set()) {
599 if (to->rem_set()->add_reference(p, 0)) {
600 _g1->schedule_popular_region_evac(to);
601 }
602 }
603 }
604 };
606 class UpdateRSetOopsIntoCSDeferred : public OopClosure {
607 G1CollectedHeap* _g1;
608 CardTableModRefBS* _ct_bs;
609 DirtyCardQueue* _dcq;
610 public:
611 UpdateRSetOopsIntoCSDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
612 _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) { }
613 virtual void do_oop(narrowOop* p) {
614 guarantee(false, "NYI");
615 }
616 virtual void do_oop(oop* p) {
617 oop obj = *p;
618 if (_g1->obj_in_cs(obj)) {
619 size_t card_index = _ct_bs->index_for(p);
620 if (_ct_bs->mark_card_deferred(card_index)) {
621 _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
622 }
623 }
624 }
625 };
627 void HRInto_G1RemSet::new_refs_iterate(OopClosure* cl) {
628 for (size_t i = 0; i < n_workers(); i++) {
629 for (int j = 0; j < _new_refs[i]->length(); j++) {
630 oop* p = _new_refs[i]->at(j);
631 cl->do_oop(p);
632 }
633 }
634 }
636 void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() {
637 guarantee( _cards_scanned != NULL, "invariant" );
638 _total_cards_scanned = 0;
639 for (uint i = 0; i < n_workers(); ++i)
640 _total_cards_scanned += _cards_scanned[i];
641 FREE_C_HEAP_ARRAY(size_t, _cards_scanned);
642 _cards_scanned = NULL;
643 // Cleanup after copy
644 #if G1_REM_SET_LOGGING
645 PrintRSClosure cl;
646 _g1->heap_region_iterate(&cl);
647 #endif
648 _g1->set_refine_cte_cl_concurrency(true);
649 cleanUpIteratorsClosure iterClosure;
650 _g1->collection_set_iterate(&iterClosure);
651 // Set all cards back to clean.
652 _g1->cleanUpCardTable();
653 if (ParallelGCThreads > 0) {
654 ConcurrentG1Refine* cg1r = _g1->concurrent_g1_refine();
655 if (cg1r->do_traversal()) {
656 cg1r->cg1rThread()->set_do_traversal(false);
657 }
658 set_par_traversal(false);
659 }
661 if (_g1->evacuation_failed()) {
662 // Restore remembered sets for the regions pointing into
663 // the collection set.
664 if (G1DeferredRSUpdate) {
665 DirtyCardQueue dcq(&_g1->dirty_card_queue_set());
666 UpdateRSetOopsIntoCSDeferred deferred_update(_g1, &dcq);
667 new_refs_iterate(&deferred_update);
668 } else {
669 UpdateRSetOopsIntoCSImmediate immediate_update(_g1);
670 new_refs_iterate(&immediate_update);
671 }
672 }
673 for (uint i = 0; i < n_workers(); i++) {
674 _new_refs[i]->clear();
675 }
677 assert(!_par_traversal_in_progress, "Invariant between iterations.");
678 }
680 class UpdateRSObjectClosure: public ObjectClosure {
681 UpdateRSOopClosure* _update_rs_oop_cl;
682 public:
683 UpdateRSObjectClosure(UpdateRSOopClosure* update_rs_oop_cl) :
684 _update_rs_oop_cl(update_rs_oop_cl) {}
685 void do_object(oop obj) {
686 obj->oop_iterate(_update_rs_oop_cl);
687 }
689 };
691 class ScrubRSClosure: public HeapRegionClosure {
692 G1CollectedHeap* _g1h;
693 BitMap* _region_bm;
694 BitMap* _card_bm;
695 CardTableModRefBS* _ctbs;
696 public:
697 ScrubRSClosure(BitMap* region_bm, BitMap* card_bm) :
698 _g1h(G1CollectedHeap::heap()),
699 _region_bm(region_bm), _card_bm(card_bm),
700 _ctbs(NULL)
701 {
702 ModRefBarrierSet* bs = _g1h->mr_bs();
703 guarantee(bs->is_a(BarrierSet::CardTableModRef), "Precondition");
704 _ctbs = (CardTableModRefBS*)bs;
705 }
707 bool doHeapRegion(HeapRegion* r) {
708 if (!r->continuesHumongous()) {
709 r->rem_set()->scrub(_ctbs, _region_bm, _card_bm);
710 }
711 return false;
712 }
713 };
715 void HRInto_G1RemSet::scrub(BitMap* region_bm, BitMap* card_bm) {
716 ScrubRSClosure scrub_cl(region_bm, card_bm);
717 _g1->heap_region_iterate(&scrub_cl);
718 }
720 void HRInto_G1RemSet::scrub_par(BitMap* region_bm, BitMap* card_bm,
721 int worker_num, int claim_val) {
722 ScrubRSClosure scrub_cl(region_bm, card_bm);
723 _g1->heap_region_par_iterate_chunked(&scrub_cl, worker_num, claim_val);
724 }
727 class ConcRefineRegionClosure: public HeapRegionClosure {
728 G1CollectedHeap* _g1h;
729 CardTableModRefBS* _ctbs;
730 ConcurrentGCThread* _cgc_thrd;
731 ConcurrentG1Refine* _cg1r;
732 unsigned _cards_processed;
733 UpdateRSOopClosure _update_rs_oop_cl;
734 public:
735 ConcRefineRegionClosure(CardTableModRefBS* ctbs,
736 ConcurrentG1Refine* cg1r,
737 HRInto_G1RemSet* g1rs) :
738 _ctbs(ctbs), _cg1r(cg1r), _cgc_thrd(cg1r->cg1rThread()),
739 _update_rs_oop_cl(g1rs), _cards_processed(0),
740 _g1h(G1CollectedHeap::heap())
741 {}
743 bool doHeapRegion(HeapRegion* r) {
744 if (!r->in_collection_set() &&
745 !r->continuesHumongous() &&
746 !r->is_young() &&
747 !r->is_survivor()) {
748 _update_rs_oop_cl.set_from(r);
749 UpdateRSObjectClosure update_rs_obj_cl(&_update_rs_oop_cl);
751 // For each run of dirty card in the region:
752 // 1) Clear the cards.
753 // 2) Process the range corresponding to the run, adding any
754 // necessary RS entries.
755 // 1 must precede 2, so that a concurrent modification redirties the
756 // card. If a processing attempt does not succeed, because it runs
757 // into an unparseable region, we will do binary search to find the
758 // beginning of the next parseable region.
759 HeapWord* startAddr = r->bottom();
760 HeapWord* endAddr = r->used_region().end();
761 HeapWord* lastAddr;
762 HeapWord* nextAddr;
764 for (nextAddr = lastAddr = startAddr;
765 nextAddr < endAddr;
766 nextAddr = lastAddr) {
767 MemRegion dirtyRegion;
769 // Get and clear dirty region from card table
770 MemRegion next_mr(nextAddr, endAddr);
771 dirtyRegion =
772 _ctbs->dirty_card_range_after_reset(
773 next_mr,
774 true, CardTableModRefBS::clean_card_val());
775 assert(dirtyRegion.start() >= nextAddr,
776 "returned region inconsistent?");
778 if (!dirtyRegion.is_empty()) {
779 HeapWord* stop_point =
780 r->object_iterate_mem_careful(dirtyRegion,
781 &update_rs_obj_cl);
782 if (stop_point == NULL) {
783 lastAddr = dirtyRegion.end();
784 _cards_processed +=
785 (int) (dirtyRegion.word_size() / CardTableModRefBS::card_size_in_words);
786 } else {
787 // We're going to skip one or more cards that we can't parse.
788 HeapWord* next_parseable_card =
789 r->next_block_start_careful(stop_point);
790 // Round this up to a card boundary.
791 next_parseable_card =
792 _ctbs->addr_for(_ctbs->byte_after_const(next_parseable_card));
793 // Now we invalidate the intervening cards so we'll see them
794 // again.
795 MemRegion remaining_dirty =
796 MemRegion(stop_point, dirtyRegion.end());
797 MemRegion skipped =
798 MemRegion(stop_point, next_parseable_card);
799 _ctbs->invalidate(skipped.intersection(remaining_dirty));
801 // Now start up again where we can parse.
802 lastAddr = next_parseable_card;
804 // Count how many we did completely.
805 _cards_processed +=
806 (stop_point - dirtyRegion.start()) /
807 CardTableModRefBS::card_size_in_words;
808 }
809 // Allow interruption at regular intervals.
810 // (Might need to make them more regular, if we get big
811 // dirty regions.)
812 if (_cgc_thrd != NULL) {
813 if (_cgc_thrd->should_yield()) {
814 _cgc_thrd->yield();
815 switch (_cg1r->get_pya()) {
816 case PYA_continue:
817 // This may have changed: re-read.
818 endAddr = r->used_region().end();
819 continue;
820 case PYA_restart: case PYA_cancel:
821 return true;
822 }
823 }
824 }
825 } else {
826 break;
827 }
828 }
829 }
830 // A good yield opportunity.
831 if (_cgc_thrd != NULL) {
832 if (_cgc_thrd->should_yield()) {
833 _cgc_thrd->yield();
834 switch (_cg1r->get_pya()) {
835 case PYA_restart: case PYA_cancel:
836 return true;
837 default:
838 break;
839 }
841 }
842 }
843 return false;
844 }
846 unsigned cards_processed() { return _cards_processed; }
847 };
850 void HRInto_G1RemSet::concurrentRefinementPass(ConcurrentG1Refine* cg1r) {
851 ConcRefineRegionClosure cr_cl(ct_bs(), cg1r, this);
852 _g1->heap_region_iterate(&cr_cl);
853 _conc_refine_traversals++;
854 _conc_refine_cards += cr_cl.cards_processed();
855 }
857 static IntHistogram out_of_histo(50, 50);
861 void HRInto_G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i) {
862 // If the card is no longer dirty, nothing to do.
863 if (*card_ptr != CardTableModRefBS::dirty_card_val()) return;
865 // Construct the region representing the card.
866 HeapWord* start = _ct_bs->addr_for(card_ptr);
867 // And find the region containing it.
868 HeapRegion* r = _g1->heap_region_containing(start);
869 if (r == NULL) {
870 guarantee(_g1->is_in_permanent(start), "Or else where?");
871 return; // Not in the G1 heap (might be in perm, for example.)
872 }
873 // Why do we have to check here whether a card is on a young region,
874 // given that we dirty young regions and, as a result, the
875 // post-barrier is supposed to filter them out and never to enqueue
876 // them? When we allocate a new region as the "allocation region" we
877 // actually dirty its cards after we release the lock, since card
878 // dirtying while holding the lock was a performance bottleneck. So,
879 // as a result, it is possible for other threads to actually
880 // allocate objects in the region (after the acquire the lock)
881 // before all the cards on the region are dirtied. This is unlikely,
882 // and it doesn't happen often, but it can happen. So, the extra
883 // check below filters out those cards.
884 if (r->is_young() || r->is_survivor()) {
885 return;
886 }
887 // While we are processing RSet buffers during the collection, we
888 // actually don't want to scan any cards on the collection set,
889 // since we don't want to update remebered sets with entries that
890 // point into the collection set, given that live objects from the
891 // collection set are about to move and such entries will be stale
892 // very soon. This change also deals with a reliability issue which
893 // involves scanning a card in the collection set and coming across
894 // an array that was being chunked and looking malformed. Note,
895 // however, that if evacuation fails, we have to scan any objects
896 // that were not moved and create any missing entries.
897 if (r->in_collection_set()) {
898 return;
899 }
901 // Should we defer it?
902 if (_cg1r->use_cache()) {
903 card_ptr = _cg1r->cache_insert(card_ptr);
904 // If it was not an eviction, nothing to do.
905 if (card_ptr == NULL) return;
907 // OK, we have to reset the card start, region, etc.
908 start = _ct_bs->addr_for(card_ptr);
909 r = _g1->heap_region_containing(start);
910 if (r == NULL) {
911 guarantee(_g1->is_in_permanent(start), "Or else where?");
912 return; // Not in the G1 heap (might be in perm, for example.)
913 }
914 guarantee(!r->is_young(), "It was evicted in the current minor cycle.");
915 }
917 HeapWord* end = _ct_bs->addr_for(card_ptr + 1);
918 MemRegion dirtyRegion(start, end);
920 #if CARD_REPEAT_HISTO
921 init_ct_freq_table(_g1->g1_reserved_obj_bytes());
922 ct_freq_note_card(_ct_bs->index_for(start));
923 #endif
925 UpdateRSOopClosure update_rs_oop_cl(this, worker_i);
926 update_rs_oop_cl.set_from(r);
927 FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r, &update_rs_oop_cl);
929 // Undirty the card.
930 *card_ptr = CardTableModRefBS::clean_card_val();
931 // We must complete this write before we do any of the reads below.
932 OrderAccess::storeload();
933 // And process it, being careful of unallocated portions of TLAB's.
934 HeapWord* stop_point =
935 r->oops_on_card_seq_iterate_careful(dirtyRegion,
936 &filter_then_update_rs_oop_cl);
937 // If stop_point is non-null, then we encountered an unallocated region
938 // (perhaps the unfilled portion of a TLAB.) For now, we'll dirty the
939 // card and re-enqueue: if we put off the card until a GC pause, then the
940 // unallocated portion will be filled in. Alternatively, we might try
941 // the full complexity of the technique used in "regular" precleaning.
942 if (stop_point != NULL) {
943 // The card might have gotten re-dirtied and re-enqueued while we
944 // worked. (In fact, it's pretty likely.)
945 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
946 *card_ptr = CardTableModRefBS::dirty_card_val();
947 MutexLockerEx x(Shared_DirtyCardQ_lock,
948 Mutex::_no_safepoint_check_flag);
949 DirtyCardQueue* sdcq =
950 JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
951 sdcq->enqueue(card_ptr);
952 }
953 } else {
954 out_of_histo.add_entry(filter_then_update_rs_oop_cl.out_of_region());
955 _conc_refine_cards++;
956 }
957 }
959 class HRRSStatsIter: public HeapRegionClosure {
960 size_t _occupied;
961 size_t _total_mem_sz;
962 size_t _max_mem_sz;
963 HeapRegion* _max_mem_sz_region;
964 public:
965 HRRSStatsIter() :
966 _occupied(0),
967 _total_mem_sz(0),
968 _max_mem_sz(0),
969 _max_mem_sz_region(NULL)
970 {}
972 bool doHeapRegion(HeapRegion* r) {
973 if (r->continuesHumongous()) return false;
974 size_t mem_sz = r->rem_set()->mem_size();
975 if (mem_sz > _max_mem_sz) {
976 _max_mem_sz = mem_sz;
977 _max_mem_sz_region = r;
978 }
979 _total_mem_sz += mem_sz;
980 size_t occ = r->rem_set()->occupied();
981 _occupied += occ;
982 return false;
983 }
984 size_t total_mem_sz() { return _total_mem_sz; }
985 size_t max_mem_sz() { return _max_mem_sz; }
986 size_t occupied() { return _occupied; }
987 HeapRegion* max_mem_sz_region() { return _max_mem_sz_region; }
988 };
990 void HRInto_G1RemSet::print_summary_info() {
991 G1CollectedHeap* g1 = G1CollectedHeap::heap();
992 ConcurrentG1RefineThread* cg1r_thrd =
993 g1->concurrent_g1_refine()->cg1rThread();
995 #if CARD_REPEAT_HISTO
996 gclog_or_tty->print_cr("\nG1 card_repeat count histogram: ");
997 gclog_or_tty->print_cr(" # of repeats --> # of cards with that number.");
998 card_repeat_count.print_on(gclog_or_tty);
999 #endif
1001 if (FILTEROUTOFREGIONCLOSURE_DOHISTOGRAMCOUNT) {
1002 gclog_or_tty->print_cr("\nG1 rem-set out-of-region histogram: ");
1003 gclog_or_tty->print_cr(" # of CS ptrs --> # of cards with that number.");
1004 out_of_histo.print_on(gclog_or_tty);
1005 }
1006 gclog_or_tty->print_cr("\n Concurrent RS processed %d cards in "
1007 "%5.2fs.",
1008 _conc_refine_cards, cg1r_thrd->vtime_accum());
1010 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
1011 jint tot_processed_buffers =
1012 dcqs.processed_buffers_mut() + dcqs.processed_buffers_rs_thread();
1013 gclog_or_tty->print_cr(" Of %d completed buffers:", tot_processed_buffers);
1014 gclog_or_tty->print_cr(" %8d (%5.1f%%) by conc RS thread.",
1015 dcqs.processed_buffers_rs_thread(),
1016 100.0*(float)dcqs.processed_buffers_rs_thread()/
1017 (float)tot_processed_buffers);
1018 gclog_or_tty->print_cr(" %8d (%5.1f%%) by mutator threads.",
1019 dcqs.processed_buffers_mut(),
1020 100.0*(float)dcqs.processed_buffers_mut()/
1021 (float)tot_processed_buffers);
1022 gclog_or_tty->print_cr(" Did %d concurrent refinement traversals.",
1023 _conc_refine_traversals);
1024 if (!G1RSBarrierUseQueue) {
1025 gclog_or_tty->print_cr(" Scanned %8.2f cards/traversal.",
1026 _conc_refine_traversals > 0 ?
1027 (float)_conc_refine_cards/(float)_conc_refine_traversals :
1028 0);
1029 }
1030 gclog_or_tty->print_cr("");
1031 if (G1UseHRIntoRS) {
1032 HRRSStatsIter blk;
1033 g1->heap_region_iterate(&blk);
1034 gclog_or_tty->print_cr(" Total heap region rem set sizes = " SIZE_FORMAT "K."
1035 " Max = " SIZE_FORMAT "K.",
1036 blk.total_mem_sz()/K, blk.max_mem_sz()/K);
1037 gclog_or_tty->print_cr(" Static structures = " SIZE_FORMAT "K,"
1038 " free_lists = " SIZE_FORMAT "K.",
1039 HeapRegionRemSet::static_mem_size()/K,
1040 HeapRegionRemSet::fl_mem_size()/K);
1041 gclog_or_tty->print_cr(" %d occupied cards represented.",
1042 blk.occupied());
1043 gclog_or_tty->print_cr(" Max sz region = [" PTR_FORMAT ", " PTR_FORMAT " )"
1044 " %s, cap = " SIZE_FORMAT "K, occ = " SIZE_FORMAT "K.",
1045 blk.max_mem_sz_region()->bottom(), blk.max_mem_sz_region()->end(),
1046 (blk.max_mem_sz_region()->popular() ? "POP" : ""),
1047 (blk.max_mem_sz_region()->rem_set()->mem_size() + K - 1)/K,
1048 (blk.max_mem_sz_region()->rem_set()->occupied() + K - 1)/K);
1049 gclog_or_tty->print_cr(" Did %d coarsenings.",
1050 HeapRegionRemSet::n_coarsenings());
1052 }
1053 }
1054 void HRInto_G1RemSet::prepare_for_verify() {
1055 if (G1HRRSFlushLogBuffersOnVerify && VerifyBeforeGC && !_g1->full_collection()) {
1056 cleanupHRRS();
1057 _g1->set_refine_cte_cl_concurrency(false);
1058 if (SafepointSynchronize::is_at_safepoint()) {
1059 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
1060 dcqs.concatenate_logs();
1061 }
1062 bool cg1r_use_cache = _cg1r->use_cache();
1063 _cg1r->set_use_cache(false);
1064 updateRS(0);
1065 _cg1r->set_use_cache(cg1r_use_cache);
1066 }
1067 }