Mercurial > jdk8-mips64-public > hotspot / changeset
changeset
6814437: G1: remove the _new_refs array
Mon, 02 Aug 2010 12:51:43 -0700
- author
- johnc
- date
- Mon, 02 Aug 2010 12:51:43 -0700
- changeset 2060
- 2d160770d2e5
- parent 2059
- 63f4675ac87d
- child 2061
- 9d7a8ab3736b
6814437: G1: remove the _new_refs array
Summary: The per-worker _new_refs array is used to hold references that point into the collection set. It is populated during RSet updating and subsequently processed. In the event of an evacuation failure it processed again to recreate the RSets of regions in the collection set. Remove the per-worker _new_refs array by processing the references directly. Use a DirtyCardQueue to hold the cards containing the references so that the RSets of regions in the collection set can be recreated when handling an evacuation failure.
Reviewed-by: iveresov, jmasa, tonyp
1.1 --- a/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp Sat Jul 31 15:10:59 2010 +0100 1.2 +++ b/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp Mon Aug 02 12:51:43 2010 -0700 1.3 @@ -339,7 +339,9 @@ 1.4 return res; 1.5 } 1.6 1.7 -void ConcurrentG1Refine::clean_up_cache(int worker_i, G1RemSet* g1rs) { 1.8 +void ConcurrentG1Refine::clean_up_cache(int worker_i, 1.9 + G1RemSet* g1rs, 1.10 + DirtyCardQueue* into_cset_dcq) { 1.11 assert(!use_cache(), "cache should be disabled"); 1.12 int start_idx; 1.13 1.14 @@ -353,7 +355,19 @@ 1.15 for (int i = start_idx; i < end_idx; i++) { 1.16 jbyte* entry = _hot_cache[i]; 1.17 if (entry != NULL) { 1.18 - g1rs->concurrentRefineOneCard(entry, worker_i); 1.19 + if (g1rs->concurrentRefineOneCard(entry, worker_i, true)) { 1.20 + // 'entry' contains references that point into the current 1.21 + // collection set. We need to record 'entry' in the DCQS 1.22 + // that's used for that purpose. 1.23 + // 1.24 + // The only time we care about recording cards that contain 1.25 + // references that point into the collection set is during 1.26 + // RSet updating while within an evacuation pause. 1.27 + // In this case worker_i should be the id of a GC worker thread 1.28 + assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause"); 1.29 + assert(worker_i < (int) DirtyCardQueueSet::num_par_ids(), "incorrect worker id"); 1.30 + into_cset_dcq->enqueue(entry); 1.31 + } 1.32 } 1.33 } 1.34 }
2.1 --- a/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp Sat Jul 31 15:10:59 2010 +0100 2.2 +++ b/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp Mon Aug 02 12:51:43 2010 -0700 2.3 @@ -1,5 +1,5 @@ 2.4 /* 2.5 - * Copyright (c) 2001, 2009, Oracle and/or its affiliates. All rights reserved. 2.6 + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 2.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 2.8 * 2.9 * This code is free software; you can redistribute it and/or modify it 2.10 @@ -184,7 +184,7 @@ 2.11 jbyte* cache_insert(jbyte* card_ptr, bool* defer); 2.12 2.13 // Process the cached entries. 2.14 - void clean_up_cache(int worker_i, G1RemSet* g1rs); 2.15 + void clean_up_cache(int worker_i, G1RemSet* g1rs, DirtyCardQueue* into_cset_dcq); 2.16 2.17 // Set up for parallel processing of the cards in the hot cache 2.18 void clear_hot_cache_claimed_index() {
3.1 --- a/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp Sat Jul 31 15:10:59 2010 +0100 3.2 +++ b/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp Mon Aug 02 12:51:43 2010 -0700 3.3 @@ -1,5 +1,5 @@ 3.4 /* 3.5 - * Copyright (c) 2001, 2009, Oracle and/or its affiliates. All rights reserved. 3.6 + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 3.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3.8 * 3.9 * This code is free software; you can redistribute it and/or modify it 3.10 @@ -178,13 +178,14 @@ 3.11 } 3.12 3.13 bool DirtyCardQueueSet:: 3.14 -apply_closure_to_completed_buffer_helper(int worker_i, 3.15 +apply_closure_to_completed_buffer_helper(CardTableEntryClosure* cl, 3.16 + int worker_i, 3.17 BufferNode* nd) { 3.18 if (nd != NULL) { 3.19 void **buf = BufferNode::make_buffer_from_node(nd); 3.20 size_t index = nd->index(); 3.21 bool b = 3.22 - DirtyCardQueue::apply_closure_to_buffer(_closure, buf, 3.23 + DirtyCardQueue::apply_closure_to_buffer(cl, buf, 3.24 index, _sz, 3.25 true, worker_i); 3.26 if (b) { 3.27 @@ -199,15 +200,22 @@ 3.28 } 3.29 } 3.30 3.31 +bool DirtyCardQueueSet::apply_closure_to_completed_buffer(CardTableEntryClosure* cl, 3.32 + int worker_i, 3.33 + int stop_at, 3.34 + bool during_pause) { 3.35 + assert(!during_pause || stop_at == 0, "Should not leave any completed buffers during a pause"); 3.36 + BufferNode* nd = get_completed_buffer(stop_at); 3.37 + bool res = apply_closure_to_completed_buffer_helper(cl, worker_i, nd); 3.38 + if (res) Atomic::inc(&_processed_buffers_rs_thread); 3.39 + return res; 3.40 +} 3.41 + 3.42 bool DirtyCardQueueSet::apply_closure_to_completed_buffer(int worker_i, 3.43 int stop_at, 3.44 - bool during_pause) 3.45 -{ 3.46 - assert(!during_pause || stop_at == 0, "Should not leave any completed buffers during a pause"); 3.47 - BufferNode* nd = get_completed_buffer(stop_at); 3.48 - bool res = apply_closure_to_completed_buffer_helper(worker_i, nd); 3.49 - if (res) Atomic::inc(&_processed_buffers_rs_thread); 3.50 - return res; 3.51 + bool during_pause) { 3.52 + return apply_closure_to_completed_buffer(_closure, worker_i, 3.53 + stop_at, during_pause); 3.54 } 3.55 3.56 void DirtyCardQueueSet::apply_closure_to_all_completed_buffers() { 3.57 @@ -222,8 +230,8 @@ 3.58 } 3.59 } 3.60 3.61 -void DirtyCardQueueSet::abandon_logs() { 3.62 - assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); 3.63 +// Deallocates any completed log buffers 3.64 +void DirtyCardQueueSet::clear() { 3.65 BufferNode* buffers_to_delete = NULL; 3.66 { 3.67 MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); 3.68 @@ -242,6 +250,12 @@ 3.69 buffers_to_delete = nd->next(); 3.70 deallocate_buffer(BufferNode::make_buffer_from_node(nd)); 3.71 } 3.72 + 3.73 +} 3.74 + 3.75 +void DirtyCardQueueSet::abandon_logs() { 3.76 + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); 3.77 + clear(); 3.78 // Since abandon is done only at safepoints, we can safely manipulate 3.79 // these queues. 3.80 for (JavaThread* t = Threads::first(); t; t = t->next()) {
4.1 --- a/src/share/vm/gc_implementation/g1/dirtyCardQueue.hpp Sat Jul 31 15:10:59 2010 +0100 4.2 +++ b/src/share/vm/gc_implementation/g1/dirtyCardQueue.hpp Mon Aug 02 12:51:43 2010 -0700 4.3 @@ -1,5 +1,5 @@ 4.4 /* 4.5 - * Copyright (c) 2001, 2009, Oracle and/or its affiliates. All rights reserved. 4.6 + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 4.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4.8 * 4.9 * This code is free software; you can redistribute it and/or modify it 4.10 @@ -123,7 +123,21 @@ 4.11 int stop_at = 0, 4.12 bool during_pause = false); 4.13 4.14 - bool apply_closure_to_completed_buffer_helper(int worker_i, 4.15 + // If there exists some completed buffer, pop it, then apply the 4.16 + // specified closure to all its elements, nulling out those elements 4.17 + // processed. If all elements are processed, returns "true". If no 4.18 + // completed buffers exist, returns false. If a completed buffer exists, 4.19 + // but is only partially completed before a "yield" happens, the 4.20 + // partially completed buffer (with its processed elements set to NULL) 4.21 + // is returned to the completed buffer set, and this call returns false. 4.22 + bool apply_closure_to_completed_buffer(CardTableEntryClosure* cl, 4.23 + int worker_i = 0, 4.24 + int stop_at = 0, 4.25 + bool during_pause = false); 4.26 + 4.27 + // Helper routine for the above. 4.28 + bool apply_closure_to_completed_buffer_helper(CardTableEntryClosure* cl, 4.29 + int worker_i, 4.30 BufferNode* nd); 4.31 4.32 BufferNode* get_completed_buffer(int stop_at); 4.33 @@ -136,6 +150,9 @@ 4.34 return &_shared_dirty_card_queue; 4.35 } 4.36 4.37 + // Deallocate any completed log buffers 4.38 + void clear(); 4.39 + 4.40 // If a full collection is happening, reset partial logs, and ignore 4.41 // completed ones: the full collection will make them all irrelevant. 4.42 void abandon_logs();
5.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Sat Jul 31 15:10:59 2010 +0100 5.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Mon Aug 02 12:51:43 2010 -0700 5.3 @@ -56,7 +56,12 @@ 5.4 _sts(sts), _g1rs(g1rs), _cg1r(cg1r), _concurrent(true) 5.5 {} 5.6 bool do_card_ptr(jbyte* card_ptr, int worker_i) { 5.7 - _g1rs->concurrentRefineOneCard(card_ptr, worker_i); 5.8 + bool oops_into_cset = _g1rs->concurrentRefineOneCard(card_ptr, worker_i, false); 5.9 + // This path is executed by the concurrent refine or mutator threads, 5.10 + // concurrently, and so we do not care if card_ptr contains references 5.11 + // that point into the collection set. 5.12 + assert(!oops_into_cset, "should be"); 5.13 + 5.14 if (_concurrent && _sts->should_yield()) { 5.15 // Caller will actually yield. 5.16 return false; 5.17 @@ -1322,6 +1327,7 @@ 5.18 SharedHeap(policy_), 5.19 _g1_policy(policy_), 5.20 _dirty_card_queue_set(false), 5.21 + _into_cset_dirty_card_queue_set(false), 5.22 _ref_processor(NULL), 5.23 _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)), 5.24 _bot_shared(NULL), 5.25 @@ -1572,6 +1578,16 @@ 5.26 Shared_DirtyCardQ_lock, 5.27 &JavaThread::dirty_card_queue_set()); 5.28 } 5.29 + 5.30 + // Initialize the card queue set used to hold cards containing 5.31 + // references into the collection set. 5.32 + _into_cset_dirty_card_queue_set.initialize(DirtyCardQ_CBL_mon, 5.33 + DirtyCardQ_FL_lock, 5.34 + -1, // never trigger processing 5.35 + -1, // no limit on length 5.36 + Shared_DirtyCardQ_lock, 5.37 + &JavaThread::dirty_card_queue_set()); 5.38 + 5.39 // In case we're keeping closure specialization stats, initialize those 5.40 // counts and that mechanism. 5.41 SpecializationStats::clear(); 5.42 @@ -1603,14 +1619,16 @@ 5.43 return _g1_committed.byte_size(); 5.44 } 5.45 5.46 -void G1CollectedHeap::iterate_dirty_card_closure(bool concurrent, 5.47 +void G1CollectedHeap::iterate_dirty_card_closure(CardTableEntryClosure* cl, 5.48 + DirtyCardQueue* into_cset_dcq, 5.49 + bool concurrent, 5.50 int worker_i) { 5.51 // Clean cards in the hot card cache 5.52 - concurrent_g1_refine()->clean_up_cache(worker_i, g1_rem_set()); 5.53 + concurrent_g1_refine()->clean_up_cache(worker_i, g1_rem_set(), into_cset_dcq); 5.54 5.55 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); 5.56 int n_completed_buffers = 0; 5.57 - while (dcqs.apply_closure_to_completed_buffer(worker_i, 0, true)) { 5.58 + while (dcqs.apply_closure_to_completed_buffer(cl, worker_i, 0, true)) { 5.59 n_completed_buffers++; 5.60 } 5.61 g1_policy()->record_update_rs_processed_buffers(worker_i, 5.62 @@ -3346,25 +3364,6 @@ 5.63 } 5.64 }; 5.65 5.66 -class UpdateRSetImmediate : public OopsInHeapRegionClosure { 5.67 -private: 5.68 - G1CollectedHeap* _g1; 5.69 - G1RemSet* _g1_rem_set; 5.70 -public: 5.71 - UpdateRSetImmediate(G1CollectedHeap* g1) : 5.72 - _g1(g1), _g1_rem_set(g1->g1_rem_set()) {} 5.73 - 5.74 - virtual void do_oop(narrowOop* p) { do_oop_work(p); } 5.75 - virtual void do_oop( oop* p) { do_oop_work(p); } 5.76 - template <class T> void do_oop_work(T* p) { 5.77 - assert(_from->is_in_reserved(p), "paranoia"); 5.78 - T heap_oop = oopDesc::load_heap_oop(p); 5.79 - if (!oopDesc::is_null(heap_oop) && !_from->is_survivor()) { 5.80 - _g1_rem_set->par_write_ref(_from, p, 0); 5.81 - } 5.82 - } 5.83 -}; 5.84 - 5.85 class UpdateRSetDeferred : public OopsInHeapRegionClosure { 5.86 private: 5.87 G1CollectedHeap* _g1; 5.88 @@ -3389,8 +3388,6 @@ 5.89 } 5.90 }; 5.91 5.92 - 5.93 - 5.94 class RemoveSelfPointerClosure: public ObjectClosure { 5.95 private: 5.96 G1CollectedHeap* _g1; 5.97 @@ -3453,7 +3450,7 @@ 5.98 }; 5.99 5.100 void G1CollectedHeap::remove_self_forwarding_pointers() { 5.101 - UpdateRSetImmediate immediate_update(_g1h); 5.102 + UpdateRSetImmediate immediate_update(_g1h->g1_rem_set()); 5.103 DirtyCardQueue dcq(&_g1h->dirty_card_queue_set()); 5.104 UpdateRSetDeferred deferred_update(_g1h, &dcq); 5.105 OopsInHeapRegionClosure *cl;
6.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Sat Jul 31 15:10:59 2010 +0100 6.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Mon Aug 02 12:51:43 2010 -0700 6.3 @@ -505,6 +505,12 @@ 6.4 // A function to check the consistency of dirty card logs. 6.5 void check_ct_logs_at_safepoint(); 6.6 6.7 + // A DirtyCardQueueSet that is used to hold cards that contain 6.8 + // references into the current collection set. This is used to 6.9 + // update the remembered sets of the regions in the collection 6.10 + // set in the event of an evacuation failure. 6.11 + DirtyCardQueueSet _into_cset_dirty_card_queue_set; 6.12 + 6.13 // After a collection pause, make the regions in the CS into free 6.14 // regions. 6.15 void free_collection_set(HeapRegion* cs_head); 6.16 @@ -661,6 +667,13 @@ 6.17 // A set of cards where updates happened during the GC 6.18 DirtyCardQueueSet& dirty_card_queue_set() { return _dirty_card_queue_set; } 6.19 6.20 + // A DirtyCardQueueSet that is used to hold cards that contain 6.21 + // references into the current collection set. This is used to 6.22 + // update the remembered sets of the regions in the collection 6.23 + // set in the event of an evacuation failure. 6.24 + DirtyCardQueueSet& into_cset_dirty_card_queue_set() 6.25 + { return _into_cset_dirty_card_queue_set; } 6.26 + 6.27 // Create a G1CollectedHeap with the specified policy. 6.28 // Must call the initialize method afterwards. 6.29 // May not return if something goes wrong. 6.30 @@ -715,7 +728,9 @@ 6.31 OrderAccess::fence(); 6.32 } 6.33 6.34 - void iterate_dirty_card_closure(bool concurrent, int worker_i); 6.35 + void iterate_dirty_card_closure(CardTableEntryClosure* cl, 6.36 + DirtyCardQueue* into_cset_dcq, 6.37 + bool concurrent, int worker_i); 6.38 6.39 // The shared block offset table array. 6.40 G1BlockOffsetSharedArray* bot_shared() const { return _bot_shared; }
7.1 --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Sat Jul 31 15:10:59 2010 +0100 7.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Mon Aug 02 12:51:43 2010 -0700 7.3 @@ -238,7 +238,6 @@ 7.4 _par_last_update_rs_processed_buffers = new double[_parallel_gc_threads]; 7.5 7.6 _par_last_scan_rs_times_ms = new double[_parallel_gc_threads]; 7.7 - _par_last_scan_new_refs_times_ms = new double[_parallel_gc_threads]; 7.8 7.9 _par_last_obj_copy_times_ms = new double[_parallel_gc_threads]; 7.10 7.11 @@ -842,7 +841,6 @@ 7.12 _par_last_update_rs_times_ms[i] = -1234.0; 7.13 _par_last_update_rs_processed_buffers[i] = -1234.0; 7.14 _par_last_scan_rs_times_ms[i] = -1234.0; 7.15 - _par_last_scan_new_refs_times_ms[i] = -1234.0; 7.16 _par_last_obj_copy_times_ms[i] = -1234.0; 7.17 _par_last_termination_times_ms[i] = -1234.0; 7.18 _par_last_termination_attempts[i] = -1234.0;
8.1 --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp Sat Jul 31 15:10:59 2010 +0100 8.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp Mon Aug 02 12:51:43 2010 -0700 8.3 @@ -63,8 +63,6 @@ 8.4 define_num_seq(mark_stack_scan) 8.5 define_num_seq(update_rs) 8.6 define_num_seq(scan_rs) 8.7 - define_num_seq(scan_new_refs) // Only for temp use; added to 8.8 - // in parallel case. 8.9 define_num_seq(obj_copy) 8.10 define_num_seq(termination) // parallel only 8.11 define_num_seq(parallel_other) // parallel only 8.12 @@ -177,7 +175,6 @@ 8.13 double* _par_last_update_rs_times_ms; 8.14 double* _par_last_update_rs_processed_buffers; 8.15 double* _par_last_scan_rs_times_ms; 8.16 - double* _par_last_scan_new_refs_times_ms; 8.17 double* _par_last_obj_copy_times_ms; 8.18 double* _par_last_termination_times_ms; 8.19 double* _par_last_termination_attempts; 8.20 @@ -933,14 +930,6 @@ 8.21 _par_last_scan_rs_times_ms[thread] = ms; 8.22 } 8.23 8.24 - void record_scan_new_refs_time(int thread, double ms) { 8.25 - _par_last_scan_new_refs_times_ms[thread] = ms; 8.26 - } 8.27 - 8.28 - double get_scan_new_refs_time(int thread) { 8.29 - return _par_last_scan_new_refs_times_ms[thread]; 8.30 - } 8.31 - 8.32 void reset_obj_copy_time(int thread) { 8.33 _par_last_obj_copy_times_ms[thread] = 0.0; 8.34 }
9.1 --- a/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp Sat Jul 31 15:10:59 2010 +0100 9.2 +++ b/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp Mon Aug 02 12:51:43 2010 -0700 9.3 @@ -1,5 +1,5 @@ 9.4 /* 9.5 - * Copyright (c) 2001, 2007, Oracle and/or its affiliates. All rights reserved. 9.6 + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 9.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 9.8 * 9.9 * This code is free software; you can redistribute it and/or modify it 9.10 @@ -37,7 +37,8 @@ 9.11 _g1->obj_in_cs(oopDesc::decode_heap_oop_not_null(heap_oop))) { 9.12 _oc->do_oop(p); 9.13 #if FILTERINTOCSCLOSURE_DOHISTOGRAMCOUNT 9.14 - _dcto_cl->incr_count(); 9.15 + if (_dcto_cl != NULL) 9.16 + _dcto_cl->incr_count(); 9.17 #endif 9.18 } 9.19 } 9.20 @@ -113,7 +114,10 @@ 9.21 if (_g1->in_cset_fast_test(obj)) { 9.22 Prefetch::write(obj->mark_addr(), 0); 9.23 Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); 9.24 + 9.25 + // Place on the references queue 9.26 _par_scan_state->push_on_queue(p); 9.27 } 9.28 } 9.29 } 9.30 +
10.1 --- a/src/share/vm/gc_implementation/g1/g1RemSet.cpp Sat Jul 31 15:10:59 2010 +0100 10.2 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.cpp Mon Aug 02 12:51:43 2010 -0700 10.3 @@ -122,23 +122,24 @@ 10.4 HRInto_G1RemSet::HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs) 10.5 : G1RemSet(g1), _ct_bs(ct_bs), _g1p(_g1->g1_policy()), 10.6 _cg1r(g1->concurrent_g1_refine()), 10.7 - _par_traversal_in_progress(false), _new_refs(NULL), 10.8 + _par_traversal_in_progress(false), 10.9 + _cset_rs_update_cl(NULL), 10.10 _cards_scanned(NULL), _total_cards_scanned(0) 10.11 { 10.12 _seq_task = new SubTasksDone(NumSeqTasks); 10.13 guarantee(n_workers() > 0, "There should be some workers"); 10.14 - _new_refs = NEW_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, n_workers()); 10.15 + _cset_rs_update_cl = NEW_C_HEAP_ARRAY(OopsInHeapRegionClosure*, n_workers()); 10.16 for (uint i = 0; i < n_workers(); i++) { 10.17 - _new_refs[i] = new (ResourceObj::C_HEAP) GrowableArray<OopOrNarrowOopStar>(8192,true); 10.18 + _cset_rs_update_cl[i] = NULL; 10.19 } 10.20 } 10.21 10.22 HRInto_G1RemSet::~HRInto_G1RemSet() { 10.23 delete _seq_task; 10.24 for (uint i = 0; i < n_workers(); i++) { 10.25 - delete _new_refs[i]; 10.26 + assert(_cset_rs_update_cl[i] == NULL, "it should be"); 10.27 } 10.28 - FREE_C_HEAP_ARRAY(GrowableArray<OopOrNarrowOopStar>*, _new_refs); 10.29 + FREE_C_HEAP_ARRAY(OopsInHeapRegionClosure*, _cset_rs_update_cl); 10.30 } 10.31 10.32 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) { 10.33 @@ -306,12 +307,45 @@ 10.34 _g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0); 10.35 } 10.36 10.37 -void HRInto_G1RemSet::updateRS(int worker_i) { 10.38 - ConcurrentG1Refine* cg1r = _g1->concurrent_g1_refine(); 10.39 +// Closure used for updating RSets and recording references that 10.40 +// point into the collection set. Only called during an 10.41 +// evacuation pause. 10.42 10.43 +class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure { 10.44 + G1RemSet* _g1rs; 10.45 + DirtyCardQueue* _into_cset_dcq; 10.46 +public: 10.47 + RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h, 10.48 + DirtyCardQueue* into_cset_dcq) : 10.49 + _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq) 10.50 + {} 10.51 + bool do_card_ptr(jbyte* card_ptr, int worker_i) { 10.52 + // The only time we care about recording cards that 10.53 + // contain references that point into the collection set 10.54 + // is during RSet updating within an evacuation pause. 10.55 + // In this case worker_i should be the id of a GC worker thread. 10.56 + assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause"); 10.57 + assert(worker_i < (int) DirtyCardQueueSet::num_par_ids(), "should be a GC worker"); 10.58 + 10.59 + if (_g1rs->concurrentRefineOneCard(card_ptr, worker_i, true)) { 10.60 + // 'card_ptr' contains references that point into the collection 10.61 + // set. We need to record the card in the DCQS 10.62 + // (G1CollectedHeap::into_cset_dirty_card_queue_set()) 10.63 + // that's used for that purpose. 10.64 + // 10.65 + // Enqueue the card 10.66 + _into_cset_dcq->enqueue(card_ptr); 10.67 + } 10.68 + return true; 10.69 + } 10.70 +}; 10.71 + 10.72 +void HRInto_G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, int worker_i) { 10.73 double start = os::elapsedTime(); 10.74 - // Apply the appropriate closure to all remaining log entries. 10.75 - _g1->iterate_dirty_card_closure(false, worker_i); 10.76 + // Apply the given closure to all remaining log entries. 10.77 + RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq); 10.78 + _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, into_cset_dcq, false, worker_i); 10.79 + 10.80 // Now there should be no dirty cards. 10.81 if (G1RSLogCheckCardTable) { 10.82 CountNonCleanMemRegionClosure cl(_g1); 10.83 @@ -405,33 +439,6 @@ 10.84 } 10.85 }; 10.86 10.87 -template <class T> void 10.88 -HRInto_G1RemSet::scanNewRefsRS_work(OopsInHeapRegionClosure* oc, 10.89 - int worker_i) { 10.90 - double scan_new_refs_start_sec = os::elapsedTime(); 10.91 - G1CollectedHeap* g1h = G1CollectedHeap::heap(); 10.92 - CardTableModRefBS* ct_bs = (CardTableModRefBS*) (g1h->barrier_set()); 10.93 - for (int i = 0; i < _new_refs[worker_i]->length(); i++) { 10.94 - T* p = (T*) _new_refs[worker_i]->at(i); 10.95 - oop obj = oopDesc::load_decode_heap_oop(p); 10.96 - // *p was in the collection set when p was pushed on "_new_refs", but 10.97 - // another thread may have processed this location from an RS, so it 10.98 - // might not point into the CS any longer. If so, it's obviously been 10.99 - // processed, and we don't need to do anything further. 10.100 - if (g1h->obj_in_cs(obj)) { 10.101 - HeapRegion* r = g1h->heap_region_containing(p); 10.102 - 10.103 - DEBUG_ONLY(HeapRegion* to = g1h->heap_region_containing(obj)); 10.104 - oc->set_region(r); 10.105 - // If "p" has already been processed concurrently, this is 10.106 - // idempotent. 10.107 - oc->do_oop(p); 10.108 - } 10.109 - } 10.110 - double scan_new_refs_time_ms = (os::elapsedTime() - scan_new_refs_start_sec) * 1000.0; 10.111 - _g1p->record_scan_new_refs_time(worker_i, scan_new_refs_time_ms); 10.112 -} 10.113 - 10.114 void HRInto_G1RemSet::cleanupHRRS() { 10.115 HeapRegionRemSet::cleanup(); 10.116 } 10.117 @@ -457,6 +464,26 @@ 10.118 count_cl.print_histo(); 10.119 } 10.120 10.121 + // We cache the value of 'oc' closure into the appropriate slot in the 10.122 + // _cset_rs_update_cl for this worker 10.123 + assert(worker_i < (int)n_workers(), "sanity"); 10.124 + _cset_rs_update_cl[worker_i] = oc; 10.125 + 10.126 + // A DirtyCardQueue that is used to hold cards containing references 10.127 + // that point into the collection set. This DCQ is associated with a 10.128 + // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal 10.129 + // circumstances (i.e. the pause successfully completes), these cards 10.130 + // are just discarded (there's no need to update the RSets of regions 10.131 + // that were in the collection set - after the pause these regions 10.132 + // are wholly 'free' of live objects. In the event of an evacuation 10.133 + // failure the cards/buffers in this queue set are: 10.134 + // * passed to the DirtyCardQueueSet that is used to manage deferred 10.135 + // RSet updates, or 10.136 + // * scanned for references that point into the collection set 10.137 + // and the RSet of the corresponding region in the collection set 10.138 + // is updated immediately. 10.139 + DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set()); 10.140 + 10.141 if (ParallelGCThreads > 0) { 10.142 // The two flags below were introduced temporarily to serialize 10.143 // the updating and scanning of remembered sets. There are some 10.144 @@ -465,12 +492,10 @@ 10.145 // conditions, we'll revert back to parallel remembered set 10.146 // updating and scanning. See CRs 6677707 and 6677708. 10.147 if (G1UseParallelRSetUpdating || (worker_i == 0)) { 10.148 - updateRS(worker_i); 10.149 - scanNewRefsRS(oc, worker_i); 10.150 + updateRS(&into_cset_dcq, worker_i); 10.151 } else { 10.152 _g1p->record_update_rs_processed_buffers(worker_i, 0.0); 10.153 _g1p->record_update_rs_time(worker_i, 0.0); 10.154 - _g1p->record_scan_new_refs_time(worker_i, 0.0); 10.155 } 10.156 if (G1UseParallelRSetScanning || (worker_i == 0)) { 10.157 scanRS(oc, worker_i); 10.158 @@ -479,10 +504,12 @@ 10.159 } 10.160 } else { 10.161 assert(worker_i == 0, "invariant"); 10.162 - updateRS(0); 10.163 - scanNewRefsRS(oc, 0); 10.164 + updateRS(&into_cset_dcq, 0); 10.165 scanRS(oc, 0); 10.166 } 10.167 + 10.168 + // We now clear the cached values of _cset_rs_update_cl for this worker 10.169 + _cset_rs_update_cl[worker_i] = NULL; 10.170 } 10.171 10.172 void HRInto_G1RemSet:: 10.173 @@ -519,49 +546,65 @@ 10.174 } 10.175 }; 10.176 10.177 -class UpdateRSetOopsIntoCSImmediate : public OopClosure { 10.178 +// This closure, applied to a DirtyCardQueueSet, is used to immediately 10.179 +// update the RSets for the regions in the CSet. For each card it iterates 10.180 +// through the oops which coincide with that card. It scans the reference 10.181 +// fields in each oop; when it finds an oop that points into the collection 10.182 +// set, the RSet for the region containing the referenced object is updated. 10.183 +// Note: _par_traversal_in_progress in the G1RemSet must be FALSE; otherwise 10.184 +// the UpdateRSetImmediate closure will cause cards to be enqueued on to 10.185 +// the DCQS that we're iterating over, causing an infinite loop. 10.186 +class UpdateRSetCardTableEntryIntoCSetClosure: public CardTableEntryClosure { 10.187 G1CollectedHeap* _g1; 10.188 + CardTableModRefBS* _ct_bs; 10.189 public: 10.190 - UpdateRSetOopsIntoCSImmediate(G1CollectedHeap* g1) : _g1(g1) { } 10.191 - virtual void do_oop(narrowOop* p) { do_oop_work(p); } 10.192 - virtual void do_oop( oop* p) { do_oop_work(p); } 10.193 - template <class T> void do_oop_work(T* p) { 10.194 - HeapRegion* to = _g1->heap_region_containing(oopDesc::load_decode_heap_oop(p)); 10.195 - if (to->in_collection_set()) { 10.196 - to->rem_set()->add_reference(p, 0); 10.197 - } 10.198 + UpdateRSetCardTableEntryIntoCSetClosure(G1CollectedHeap* g1, 10.199 + CardTableModRefBS* bs): 10.200 + _g1(g1), _ct_bs(bs) 10.201 + { } 10.202 + 10.203 + bool do_card_ptr(jbyte* card_ptr, int worker_i) { 10.204 + // Construct the region representing the card. 10.205 + HeapWord* start = _ct_bs->addr_for(card_ptr); 10.206 + // And find the region containing it. 10.207 + HeapRegion* r = _g1->heap_region_containing(start); 10.208 + assert(r != NULL, "unexpected null"); 10.209 + 10.210 + // Scan oops in the card looking for references into the collection set 10.211 + HeapWord* end = _ct_bs->addr_for(card_ptr + 1); 10.212 + MemRegion scanRegion(start, end); 10.213 + 10.214 + UpdateRSetImmediate update_rs_cl(_g1->g1_rem_set()); 10.215 + FilterIntoCSClosure update_rs_cset_oop_cl(NULL, _g1, &update_rs_cl); 10.216 + FilterOutOfRegionClosure filter_then_update_rs_cset_oop_cl(r, &update_rs_cset_oop_cl); 10.217 + 10.218 + // We can pass false as the "filter_young" parameter here as: 10.219 + // * we should be in a STW pause, 10.220 + // * the DCQS to which this closure is applied is used to hold 10.221 + // references that point into the collection set from the prior 10.222 + // RSet updating, 10.223 + // * the post-write barrier shouldn't be logging updates to young 10.224 + // regions (but there is a situation where this can happen - see 10.225 + // the comment in HRInto_G1RemSet::concurrentRefineOneCard below - 10.226 + // that should not be applicable here), and 10.227 + // * during actual RSet updating, the filtering of cards in young 10.228 + // regions in HeapRegion::oops_on_card_seq_iterate_careful is 10.229 + // employed. 10.230 + // As a result, when this closure is applied to "refs into cset" 10.231 + // DCQS, we shouldn't see any cards in young regions. 10.232 + update_rs_cl.set_region(r); 10.233 + HeapWord* stop_point = 10.234 + r->oops_on_card_seq_iterate_careful(scanRegion, 10.235 + &filter_then_update_rs_cset_oop_cl, 10.236 + false /* filter_young */); 10.237 + 10.238 + // Since this is performed in the event of an evacuation failure, we 10.239 + // we shouldn't see a non-null stop point 10.240 + assert(stop_point == NULL, "saw an unallocated region"); 10.241 + return true; 10.242 } 10.243 }; 10.244 10.245 -class UpdateRSetOopsIntoCSDeferred : public OopClosure { 10.246 - G1CollectedHeap* _g1; 10.247 - CardTableModRefBS* _ct_bs; 10.248 - DirtyCardQueue* _dcq; 10.249 -public: 10.250 - UpdateRSetOopsIntoCSDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) : 10.251 - _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) { } 10.252 - virtual void do_oop(narrowOop* p) { do_oop_work(p); } 10.253 - virtual void do_oop( oop* p) { do_oop_work(p); } 10.254 - template <class T> void do_oop_work(T* p) { 10.255 - oop obj = oopDesc::load_decode_heap_oop(p); 10.256 - if (_g1->obj_in_cs(obj)) { 10.257 - size_t card_index = _ct_bs->index_for(p); 10.258 - if (_ct_bs->mark_card_deferred(card_index)) { 10.259 - _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index)); 10.260 - } 10.261 - } 10.262 - } 10.263 -}; 10.264 - 10.265 -template <class T> void HRInto_G1RemSet::new_refs_iterate_work(OopClosure* cl) { 10.266 - for (size_t i = 0; i < n_workers(); i++) { 10.267 - for (int j = 0; j < _new_refs[i]->length(); j++) { 10.268 - T* p = (T*) _new_refs[i]->at(j); 10.269 - cl->do_oop(p); 10.270 - } 10.271 - } 10.272 -} 10.273 - 10.274 void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() { 10.275 guarantee( _cards_scanned != NULL, "invariant" ); 10.276 _total_cards_scanned = 0; 10.277 @@ -584,21 +627,38 @@ 10.278 set_par_traversal(false); 10.279 } 10.280 10.281 + DirtyCardQueueSet& into_cset_dcqs = _g1->into_cset_dirty_card_queue_set(); 10.282 + int into_cset_n_buffers = into_cset_dcqs.completed_buffers_num(); 10.283 + 10.284 if (_g1->evacuation_failed()) { 10.285 - // Restore remembered sets for the regions pointing into 10.286 - // the collection set. 10.287 + // Restore remembered sets for the regions pointing into the collection set. 10.288 + 10.289 if (G1DeferredRSUpdate) { 10.290 - DirtyCardQueue dcq(&_g1->dirty_card_queue_set()); 10.291 - UpdateRSetOopsIntoCSDeferred deferred_update(_g1, &dcq); 10.292 - new_refs_iterate(&deferred_update); 10.293 + // If deferred RS updates are enabled then we just need to transfer 10.294 + // the completed buffers from (a) the DirtyCardQueueSet used to hold 10.295 + // cards that contain references that point into the collection set 10.296 + // to (b) the DCQS used to hold the deferred RS updates 10.297 + _g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs); 10.298 } else { 10.299 - UpdateRSetOopsIntoCSImmediate immediate_update(_g1); 10.300 - new_refs_iterate(&immediate_update); 10.301 + 10.302 + CardTableModRefBS* bs = (CardTableModRefBS*)_g1->barrier_set(); 10.303 + UpdateRSetCardTableEntryIntoCSetClosure update_rs_cset_immediate(_g1, bs); 10.304 + 10.305 + int n_completed_buffers = 0; 10.306 + while (into_cset_dcqs.apply_closure_to_completed_buffer(&update_rs_cset_immediate, 10.307 + 0, 0, true)) { 10.308 + n_completed_buffers++; 10.309 + } 10.310 + assert(n_completed_buffers == into_cset_n_buffers, "missed some buffers"); 10.311 } 10.312 } 10.313 - for (uint i = 0; i < n_workers(); i++) { 10.314 - _new_refs[i]->clear(); 10.315 - } 10.316 + 10.317 + // Free any completed buffers in the DirtyCardQueueSet used to hold cards 10.318 + // which contain references that point into the collection. 10.319 + _g1->into_cset_dirty_card_queue_set().clear(); 10.320 + assert(_g1->into_cset_dirty_card_queue_set().completed_buffers_num() == 0, 10.321 + "all buffers should be freed"); 10.322 + _g1->into_cset_dirty_card_queue_set().clear_n_completed_buffers(); 10.323 10.324 assert(!_par_traversal_in_progress, "Invariant between iterations."); 10.325 } 10.326 @@ -652,7 +712,43 @@ 10.327 10.328 static IntHistogram out_of_histo(50, 50); 10.329 10.330 -void HRInto_G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i) { 10.331 +class TriggerClosure : public OopClosure { 10.332 + bool _trigger; 10.333 +public: 10.334 + TriggerClosure() : _trigger(false) { } 10.335 + bool value() const { return _trigger; } 10.336 + template <class T> void do_oop_nv(T* p) { _trigger = true; } 10.337 + virtual void do_oop(oop* p) { do_oop_nv(p); } 10.338 + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } 10.339 +}; 10.340 + 10.341 +class InvokeIfNotTriggeredClosure: public OopClosure { 10.342 + TriggerClosure* _t; 10.343 + OopClosure* _oc; 10.344 +public: 10.345 + InvokeIfNotTriggeredClosure(TriggerClosure* t, OopClosure* oc): 10.346 + _t(t), _oc(oc) { } 10.347 + template <class T> void do_oop_nv(T* p) { 10.348 + if (!_t->value()) _oc->do_oop(p); 10.349 + } 10.350 + virtual void do_oop(oop* p) { do_oop_nv(p); } 10.351 + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } 10.352 +}; 10.353 + 10.354 +class Mux2Closure : public OopClosure { 10.355 + OopClosure* _c1; 10.356 + OopClosure* _c2; 10.357 +public: 10.358 + Mux2Closure(OopClosure *c1, OopClosure *c2) : _c1(c1), _c2(c2) { } 10.359 + template <class T> void do_oop_nv(T* p) { 10.360 + _c1->do_oop(p); _c2->do_oop(p); 10.361 + } 10.362 + virtual void do_oop(oop* p) { do_oop_nv(p); } 10.363 + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } 10.364 +}; 10.365 + 10.366 +bool HRInto_G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i, 10.367 + bool check_for_refs_into_cset) { 10.368 // Construct the region representing the card. 10.369 HeapWord* start = _ct_bs->addr_for(card_ptr); 10.370 // And find the region containing it. 10.371 @@ -669,7 +765,16 @@ 10.372 10.373 UpdateRSOopClosure update_rs_oop_cl(this, worker_i); 10.374 update_rs_oop_cl.set_from(r); 10.375 - FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r, &update_rs_oop_cl); 10.376 + 10.377 + TriggerClosure trigger_cl; 10.378 + FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl); 10.379 + InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl); 10.380 + Mux2Closure mux(&invoke_cl, &update_rs_oop_cl); 10.381 + 10.382 + FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r, 10.383 + (check_for_refs_into_cset ? 10.384 + (OopClosure*)&mux : 10.385 + (OopClosure*)&update_rs_oop_cl)); 10.386 10.387 // Undirty the card. 10.388 *card_ptr = CardTableModRefBS::clean_card_val(); 10.389 @@ -717,11 +822,18 @@ 10.390 out_of_histo.add_entry(filter_then_update_rs_oop_cl.out_of_region()); 10.391 _conc_refine_cards++; 10.392 } 10.393 + 10.394 + return trigger_cl.value(); 10.395 } 10.396 10.397 -void HRInto_G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i) { 10.398 +bool HRInto_G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i, 10.399 + bool check_for_refs_into_cset) { 10.400 // If the card is no longer dirty, nothing to do. 10.401 - if (*card_ptr != CardTableModRefBS::dirty_card_val()) return; 10.402 + if (*card_ptr != CardTableModRefBS::dirty_card_val()) { 10.403 + // No need to return that this card contains refs that point 10.404 + // into the collection set. 10.405 + return false; 10.406 + } 10.407 10.408 // Construct the region representing the card. 10.409 HeapWord* start = _ct_bs->addr_for(card_ptr); 10.410 @@ -729,7 +841,9 @@ 10.411 HeapRegion* r = _g1->heap_region_containing(start); 10.412 if (r == NULL) { 10.413 guarantee(_g1->is_in_permanent(start), "Or else where?"); 10.414 - return; // Not in the G1 heap (might be in perm, for example.) 10.415 + // Again no need to return that this card contains refs that 10.416 + // point into the collection set. 10.417 + return false; // Not in the G1 heap (might be in perm, for example.) 10.418 } 10.419 // Why do we have to check here whether a card is on a young region, 10.420 // given that we dirty young regions and, as a result, the 10.421 @@ -743,7 +857,7 @@ 10.422 // and it doesn't happen often, but it can happen. So, the extra 10.423 // check below filters out those cards. 10.424 if (r->is_young()) { 10.425 - return; 10.426 + return false; 10.427 } 10.428 // While we are processing RSet buffers during the collection, we 10.429 // actually don't want to scan any cards on the collection set, 10.430 @@ -756,7 +870,7 @@ 10.431 // however, that if evacuation fails, we have to scan any objects 10.432 // that were not moved and create any missing entries. 10.433 if (r->in_collection_set()) { 10.434 - return; 10.435 + return false; 10.436 } 10.437 10.438 // Should we defer processing the card? 10.439 @@ -797,8 +911,14 @@ 10.440 // cache. 10.441 // Immediately process res; no need to process card_ptr. 10.442 10.443 + 10.444 jbyte* res = card_ptr; 10.445 bool defer = false; 10.446 + 10.447 + // This gets set to true if the card being refined has references 10.448 + // that point into the collection set. 10.449 + bool oops_into_cset = false; 10.450 + 10.451 if (_cg1r->use_cache()) { 10.452 jbyte* res = _cg1r->cache_insert(card_ptr, &defer); 10.453 if (res != NULL && (res != card_ptr || defer)) { 10.454 @@ -815,14 +935,31 @@ 10.455 // Process card pointer we get back from the hot card cache. This 10.456 // will check whether the region containing the card is young 10.457 // _after_ checking that the region has been allocated from. 10.458 - concurrentRefineOneCard_impl(res, worker_i); 10.459 + oops_into_cset = concurrentRefineOneCard_impl(res, worker_i, 10.460 + false /* check_for_refs_into_cset */); 10.461 + // The above call to concurrentRefineOneCard_impl is only 10.462 + // performed if the hot card cache is enabled. This cache is 10.463 + // disabled during an evacuation pause - which is the only 10.464 + // time when we need know if the card contains references 10.465 + // that point into the collection set. Also when the hot card 10.466 + // cache is enabled, this code is executed by the concurrent 10.467 + // refine threads - rather than the GC worker threads - and 10.468 + // concurrentRefineOneCard_impl will return false. 10.469 + assert(!oops_into_cset, "should not see true here"); 10.470 } 10.471 } 10.472 } 10.473 10.474 if (!defer) { 10.475 - concurrentRefineOneCard_impl(card_ptr, worker_i); 10.476 + oops_into_cset = 10.477 + concurrentRefineOneCard_impl(card_ptr, worker_i, check_for_refs_into_cset); 10.478 + // We should only be detecting that the card contains references 10.479 + // that point into the collection set if the current thread is 10.480 + // a GC worker thread. 10.481 + assert(!oops_into_cset || SafepointSynchronize::is_at_safepoint(), 10.482 + "invalid result at non safepoint"); 10.483 } 10.484 + return oops_into_cset; 10.485 } 10.486 10.487 class HRRSStatsIter: public HeapRegionClosure { 10.488 @@ -920,6 +1057,7 @@ 10.489 10.490 } 10.491 } 10.492 + 10.493 void HRInto_G1RemSet::prepare_for_verify() { 10.494 if (G1HRRSFlushLogBuffersOnVerify && 10.495 (VerifyBeforeGC || VerifyAfterGC) 10.496 @@ -932,7 +1070,9 @@ 10.497 } 10.498 bool cg1r_use_cache = _cg1r->use_cache(); 10.499 _cg1r->set_use_cache(false); 10.500 - updateRS(0); 10.501 + DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set()); 10.502 + updateRS(&into_cset_dcq, 0); 10.503 + _g1->into_cset_dirty_card_queue_set().clear(); 10.504 _cg1r->set_use_cache(cg1r_use_cache); 10.505 10.506 assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed");
11.1 --- a/src/share/vm/gc_implementation/g1/g1RemSet.hpp Sat Jul 31 15:10:59 2010 +0100 11.2 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.hpp Mon Aug 02 12:51:43 2010 -0700 11.3 @@ -1,5 +1,5 @@ 11.4 /* 11.5 - * Copyright (c) 2001, 2009, Oracle and/or its affiliates. All rights reserved. 11.6 + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 11.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 11.8 * 11.9 * This code is free software; you can redistribute it and/or modify it 11.10 @@ -83,7 +83,13 @@ 11.11 // Refine the card corresponding to "card_ptr". If "sts" is non-NULL, 11.12 // join and leave around parts that must be atomic wrt GC. (NULL means 11.13 // being done at a safepoint.) 11.14 - virtual void concurrentRefineOneCard(jbyte* card_ptr, int worker_i) {} 11.15 + // With some implementations of this routine, when check_for_refs_into_cset 11.16 + // is true, a true result may be returned if the given card contains oops 11.17 + // that have references into the current collection set. 11.18 + virtual bool concurrentRefineOneCard(jbyte* card_ptr, int worker_i, 11.19 + bool check_for_refs_into_cset) { 11.20 + return false; 11.21 + } 11.22 11.23 // Print any relevant summary info. 11.24 virtual void print_summary_info() {} 11.25 @@ -143,23 +149,21 @@ 11.26 size_t _total_cards_scanned; 11.27 11.28 // _par_traversal_in_progress is "true" iff a parallel traversal is in 11.29 - // progress. If so, then cards added to remembered sets should also have 11.30 - // their references into the collection summarized in "_new_refs". 11.31 + // progress. 11.32 bool _par_traversal_in_progress; 11.33 void set_par_traversal(bool b) { _par_traversal_in_progress = b; } 11.34 - GrowableArray<OopOrNarrowOopStar>** _new_refs; 11.35 - template <class T> void new_refs_iterate_work(OopClosure* cl); 11.36 - void new_refs_iterate(OopClosure* cl) { 11.37 - if (UseCompressedOops) { 11.38 - new_refs_iterate_work<narrowOop>(cl); 11.39 - } else { 11.40 - new_refs_iterate_work<oop>(cl); 11.41 - } 11.42 - } 11.43 + 11.44 + // Used for caching the closure that is responsible for scanning 11.45 + // references into the collection set. 11.46 + OopsInHeapRegionClosure** _cset_rs_update_cl; 11.47 11.48 // The routine that performs the actual work of refining a dirty 11.49 // card. 11.50 - void concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i); 11.51 + // If check_for_refs_into_refs is true then a true result is returned 11.52 + // if the card contains oops that have references into the current 11.53 + // collection set. 11.54 + bool concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i, 11.55 + bool check_for_refs_into_cset); 11.56 11.57 protected: 11.58 template <class T> void write_ref_nv(HeapRegion* from, T* p); 11.59 @@ -188,7 +192,7 @@ 11.60 scanNewRefsRS_work<oop>(oc, worker_i); 11.61 } 11.62 } 11.63 - void updateRS(int worker_i); 11.64 + void updateRS(DirtyCardQueue* into_cset_dcq, int worker_i); 11.65 HeapRegion* calculateStartRegion(int i); 11.66 11.67 HRInto_G1RemSet* as_HRInto_G1RemSet() { return this; } 11.68 @@ -219,7 +223,11 @@ 11.69 void scrub_par(BitMap* region_bm, BitMap* card_bm, 11.70 int worker_num, int claim_val); 11.71 11.72 - virtual void concurrentRefineOneCard(jbyte* card_ptr, int worker_i); 11.73 + // If check_for_refs_into_cset is true then a true result is returned 11.74 + // if the card contains oops that have references into the current 11.75 + // collection set. 11.76 + virtual bool concurrentRefineOneCard(jbyte* card_ptr, int worker_i, 11.77 + bool check_for_refs_into_cset); 11.78 11.79 virtual void print_summary_info(); 11.80 virtual void prepare_for_verify(); 11.81 @@ -265,3 +273,16 @@ 11.82 // bool idempotent() { return true; } 11.83 bool apply_to_weak_ref_discovered_field() { return true; } 11.84 }; 11.85 + 11.86 +class UpdateRSetImmediate: public OopsInHeapRegionClosure { 11.87 +private: 11.88 + G1RemSet* _g1_rem_set; 11.89 + 11.90 + template <class T> void do_oop_work(T* p); 11.91 +public: 11.92 + UpdateRSetImmediate(G1RemSet* rs) : 11.93 + _g1_rem_set(rs) {} 11.94 + 11.95 + virtual void do_oop(narrowOop* p) { do_oop_work(p); } 11.96 + virtual void do_oop( oop* p) { do_oop_work(p); } 11.97 +};
12.1 --- a/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp Sat Jul 31 15:10:59 2010 +0100 12.2 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp Mon Aug 02 12:51:43 2010 -0700 12.3 @@ -1,5 +1,5 @@ 12.4 /* 12.5 - * Copyright (c) 2001, 2009, Oracle and/or its affiliates. All rights reserved. 12.6 + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 12.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 12.8 * 12.9 * This code is free software; you can redistribute it and/or modify it 12.10 @@ -56,19 +56,25 @@ 12.11 assert(Universe::heap()->is_in_reserved(obj), "must be in heap"); 12.12 } 12.13 #endif // ASSERT 12.14 - assert(from == NULL || from->is_in_reserved(p), 12.15 - "p is not in from"); 12.16 + 12.17 + assert(from == NULL || from->is_in_reserved(p), "p is not in from"); 12.18 + 12.19 HeapRegion* to = _g1->heap_region_containing(obj); 12.20 // The test below could be optimized by applying a bit op to to and from. 12.21 if (to != NULL && from != NULL && from != to) { 12.22 - // There is a tricky infinite loop if we keep pushing 12.23 - // self forwarding pointers onto our _new_refs list. 12.24 // The _par_traversal_in_progress flag is true during the collection pause, 12.25 - // false during the evacuation failure handing. 12.26 + // false during the evacuation failure handing. This should avoid a 12.27 + // potential loop if we were to add the card containing 'p' to the DCQS 12.28 + // that's used to regenerate the remembered sets for the collection set, 12.29 + // in the event of an evacuation failure, here. The UpdateRSImmediate 12.30 + // closure will eventally call this routine. 12.31 if (_par_traversal_in_progress && 12.32 to->in_collection_set() && !self_forwarded(obj)) { 12.33 - _new_refs[tid]->push((void*)p); 12.34 - // Deferred updates to the Cset are either discarded (in the normal case), 12.35 + 12.36 + assert(_cset_rs_update_cl[tid] != NULL, "should have been set already"); 12.37 + _cset_rs_update_cl[tid]->do_oop(p); 12.38 + 12.39 + // Deferred updates to the CSet are either discarded (in the normal case), 12.40 // or processed (if an evacuation failure occurs) at the end 12.41 // of the collection. 12.42 // See HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do(). 12.43 @@ -89,3 +95,12 @@ 12.44 assert(_from != NULL, "from region must be non-NULL"); 12.45 _rs->par_write_ref(_from, p, _worker_i); 12.46 } 12.47 + 12.48 +template <class T> inline void UpdateRSetImmediate::do_oop_work(T* p) { 12.49 + assert(_from->is_in_reserved(p), "paranoia"); 12.50 + T heap_oop = oopDesc::load_heap_oop(p); 12.51 + if (!oopDesc::is_null(heap_oop) && !_from->is_survivor()) { 12.52 + _g1_rem_set->par_write_ref(_from, p, 0); 12.53 + } 12.54 +} 12.55 +
13.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.cpp Sat Jul 31 15:10:59 2010 +0100 13.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp Mon Aug 02 12:51:43 2010 -0700 13.3 @@ -683,6 +683,8 @@ 13.4 return NULL; 13.5 } 13.6 13.7 + assert(!is_young(), "check value of filter_young"); 13.8 + 13.9 // We used to use "block_start_careful" here. But we're actually happy 13.10 // to update the BOT while we do this... 13.11 HeapWord* cur = block_start(mr.start());
14.1 --- a/src/share/vm/gc_implementation/includeDB_gc_g1 Sat Jul 31 15:10:59 2010 +0100 14.2 +++ b/src/share/vm/gc_implementation/includeDB_gc_g1 Mon Aug 02 12:51:43 2010 -0700 14.3 @@ -1,5 +1,5 @@ 14.4 // 14.5 -// Copyright (c) 2004, 2009, Oracle and/or its affiliates. All rights reserved. 14.6 +// Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. 14.7 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 14.8 // 14.9 // This code is free software; you can redistribute it and/or modify it 14.10 @@ -241,6 +241,7 @@ 14.11 14.12 g1MMUTracker.hpp debug.hpp 14.13 g1MMUTracker.hpp allocation.hpp 14.14 + 14.15 g1RemSet.cpp bufferingOopClosure.hpp 14.16 g1RemSet.cpp concurrentG1Refine.hpp 14.17 g1RemSet.cpp concurrentG1RefineThread.hpp
