1.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp Wed Jan 19 13:04:37 2011 -0800
1.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp Wed Jan 19 19:30:42 2011 -0500
1.3 @@ -458,6 +458,7 @@
1.4 _marking_task_overhead(1.0),
1.5 _cleanup_sleep_factor(0.0),
1.6 _cleanup_task_overhead(1.0),
1.7 + _cleanup_list("Cleanup List"),
1.8 _region_bm(max_regions, false /* in_resource_area*/),
1.9 _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >>
1.10 CardTableModRefBS::card_shift,
1.11 @@ -521,12 +522,6 @@
1.12 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
1.13 satb_qs.set_buffer_size(G1SATBBufferSize);
1.14
1.15 - int size = (int) MAX2(ParallelGCThreads, (size_t)1);
1.16 - _par_cleanup_thread_state = NEW_C_HEAP_ARRAY(ParCleanupThreadState*, size);
1.17 - for (int i = 0 ; i < size; i++) {
1.18 - _par_cleanup_thread_state[i] = new ParCleanupThreadState;
1.19 - }
1.20 -
1.21 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num);
1.22 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num);
1.23
1.24 @@ -711,11 +706,6 @@
1.25 }
1.26
1.27 ConcurrentMark::~ConcurrentMark() {
1.28 - int size = (int) MAX2(ParallelGCThreads, (size_t)1);
1.29 - for (int i = 0; i < size; i++) delete _par_cleanup_thread_state[i];
1.30 - FREE_C_HEAP_ARRAY(ParCleanupThreadState*,
1.31 - _par_cleanup_thread_state);
1.32 -
1.33 for (int i = 0; i < (int) _max_task_num; ++i) {
1.34 delete _task_queues->queue(i);
1.35 delete _tasks[i];
1.36 @@ -1510,21 +1500,20 @@
1.37 size_t _max_live_bytes;
1.38 size_t _regions_claimed;
1.39 size_t _freed_bytes;
1.40 - size_t _cleared_h_regions;
1.41 - size_t _freed_regions;
1.42 - UncleanRegionList* _unclean_region_list;
1.43 + FreeRegionList _local_cleanup_list;
1.44 + HumongousRegionSet _humongous_proxy_set;
1.45 double _claimed_region_time;
1.46 double _max_region_time;
1.47
1.48 public:
1.49 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1.50 - UncleanRegionList* list,
1.51 int worker_num);
1.52 size_t freed_bytes() { return _freed_bytes; }
1.53 - size_t cleared_h_regions() { return _cleared_h_regions; }
1.54 - size_t freed_regions() { return _freed_regions; }
1.55 - UncleanRegionList* unclean_region_list() {
1.56 - return _unclean_region_list;
1.57 + FreeRegionList* local_cleanup_list() {
1.58 + return &_local_cleanup_list;
1.59 + }
1.60 + HumongousRegionSet* humongous_proxy_set() {
1.61 + return &_humongous_proxy_set;
1.62 }
1.63
1.64 bool doHeapRegion(HeapRegion *r);
1.65 @@ -1537,25 +1526,22 @@
1.66
1.67 class G1ParNoteEndTask: public AbstractGangTask {
1.68 friend class G1NoteEndOfConcMarkClosure;
1.69 +
1.70 protected:
1.71 G1CollectedHeap* _g1h;
1.72 size_t _max_live_bytes;
1.73 size_t _freed_bytes;
1.74 - ConcurrentMark::ParCleanupThreadState** _par_cleanup_thread_state;
1.75 + FreeRegionList* _cleanup_list;
1.76 +
1.77 public:
1.78 G1ParNoteEndTask(G1CollectedHeap* g1h,
1.79 - ConcurrentMark::ParCleanupThreadState**
1.80 - par_cleanup_thread_state) :
1.81 + FreeRegionList* cleanup_list) :
1.82 AbstractGangTask("G1 note end"), _g1h(g1h),
1.83 - _max_live_bytes(0), _freed_bytes(0),
1.84 - _par_cleanup_thread_state(par_cleanup_thread_state)
1.85 - {}
1.86 + _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list) { }
1.87
1.88 void work(int i) {
1.89 double start = os::elapsedTime();
1.90 - G1NoteEndOfConcMarkClosure g1_note_end(_g1h,
1.91 - &_par_cleanup_thread_state[i]->list,
1.92 - i);
1.93 + G1NoteEndOfConcMarkClosure g1_note_end(_g1h, i);
1.94 if (G1CollectedHeap::use_parallel_gc_threads()) {
1.95 _g1h->heap_region_par_iterate_chunked(&g1_note_end, i,
1.96 HeapRegion::NoteEndClaimValue);
1.97 @@ -1564,14 +1550,18 @@
1.98 }
1.99 assert(g1_note_end.complete(), "Shouldn't have yielded!");
1.100
1.101 - // Now finish up freeing the current thread's regions.
1.102 - _g1h->finish_free_region_work(g1_note_end.freed_bytes(),
1.103 - g1_note_end.cleared_h_regions(),
1.104 - 0, NULL);
1.105 + // Now update the lists
1.106 + _g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(),
1.107 + NULL /* free_list */,
1.108 + g1_note_end.humongous_proxy_set(),
1.109 + true /* par */);
1.110 {
1.111 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1.112 _max_live_bytes += g1_note_end.max_live_bytes();
1.113 _freed_bytes += g1_note_end.freed_bytes();
1.114 +
1.115 + _cleanup_list->add_as_tail(g1_note_end.local_cleanup_list());
1.116 + assert(g1_note_end.local_cleanup_list()->is_empty(), "post-condition");
1.117 }
1.118 double end = os::elapsedTime();
1.119 if (G1PrintParCleanupStats) {
1.120 @@ -1612,30 +1602,28 @@
1.121
1.122 G1NoteEndOfConcMarkClosure::
1.123 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1.124 - UncleanRegionList* list,
1.125 int worker_num)
1.126 : _g1(g1), _worker_num(worker_num),
1.127 _max_live_bytes(0), _regions_claimed(0),
1.128 - _freed_bytes(0), _cleared_h_regions(0), _freed_regions(0),
1.129 + _freed_bytes(0),
1.130 _claimed_region_time(0.0), _max_region_time(0.0),
1.131 - _unclean_region_list(list)
1.132 -{}
1.133 -
1.134 -bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *r) {
1.135 + _local_cleanup_list("Local Cleanup List"),
1.136 + _humongous_proxy_set("Local Cleanup Humongous Proxy Set") { }
1.137 +
1.138 +bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *hr) {
1.139 // We use a claim value of zero here because all regions
1.140 // were claimed with value 1 in the FinalCount task.
1.141 - r->reset_gc_time_stamp();
1.142 - if (!r->continuesHumongous()) {
1.143 + hr->reset_gc_time_stamp();
1.144 + if (!hr->continuesHumongous()) {
1.145 double start = os::elapsedTime();
1.146 _regions_claimed++;
1.147 - r->note_end_of_marking();
1.148 - _max_live_bytes += r->max_live_bytes();
1.149 - _g1->free_region_if_totally_empty_work(r,
1.150 - _freed_bytes,
1.151 - _cleared_h_regions,
1.152 - _freed_regions,
1.153 - _unclean_region_list,
1.154 - true /*par*/);
1.155 + hr->note_end_of_marking();
1.156 + _max_live_bytes += hr->max_live_bytes();
1.157 + _g1->free_region_if_totally_empty(hr,
1.158 + &_freed_bytes,
1.159 + &_local_cleanup_list,
1.160 + &_humongous_proxy_set,
1.161 + true /* par */);
1.162 double region_time = (os::elapsedTime() - start);
1.163 _claimed_region_time += region_time;
1.164 if (region_time > _max_region_time) _max_region_time = region_time;
1.165 @@ -1655,6 +1643,8 @@
1.166 return;
1.167 }
1.168
1.169 + g1h->verify_region_sets_optional();
1.170 +
1.171 if (VerifyDuringGC) {
1.172 HandleMark hm; // handle scope
1.173 gclog_or_tty->print(" VerifyDuringGC:(before)");
1.174 @@ -1719,7 +1709,7 @@
1.175
1.176 // Note end of marking in all heap regions.
1.177 double note_end_start = os::elapsedTime();
1.178 - G1ParNoteEndTask g1_par_note_end_task(g1h, _par_cleanup_thread_state);
1.179 + G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list);
1.180 if (G1CollectedHeap::use_parallel_gc_threads()) {
1.181 int n_workers = g1h->workers()->total_workers();
1.182 g1h->set_par_threads(n_workers);
1.183 @@ -1731,9 +1721,14 @@
1.184 } else {
1.185 g1_par_note_end_task.work(0);
1.186 }
1.187 - g1h->set_unclean_regions_coming(true);
1.188 +
1.189 + if (!cleanup_list_is_empty()) {
1.190 + // The cleanup list is not empty, so we'll have to process it
1.191 + // concurrently. Notify anyone else that might be wanting free
1.192 + // regions that there will be more free regions coming soon.
1.193 + g1h->set_free_regions_coming();
1.194 + }
1.195 double note_end_end = os::elapsedTime();
1.196 - // Tell the mutators that there might be unclean regions coming...
1.197 if (G1PrintParCleanupStats) {
1.198 gclog_or_tty->print_cr(" note end of marking: %8.3f ms.",
1.199 (note_end_end - note_end_start)*1000.0);
1.200 @@ -1799,33 +1794,63 @@
1.201 /* silent */ false,
1.202 /* prev marking */ true);
1.203 }
1.204 +
1.205 + g1h->verify_region_sets_optional();
1.206 }
1.207
1.208 void ConcurrentMark::completeCleanup() {
1.209 - // A full collection intervened.
1.210 if (has_aborted()) return;
1.211
1.212 - int first = 0;
1.213 - int last = (int)MAX2(ParallelGCThreads, (size_t)1);
1.214 - for (int t = 0; t < last; t++) {
1.215 - UncleanRegionList* list = &_par_cleanup_thread_state[t]->list;
1.216 - assert(list->well_formed(), "Inv");
1.217 - HeapRegion* hd = list->hd();
1.218 - while (hd != NULL) {
1.219 - // Now finish up the other stuff.
1.220 - hd->rem_set()->clear();
1.221 - HeapRegion* next_hd = hd->next_from_unclean_list();
1.222 - (void)list->pop();
1.223 - assert(list->hd() == next_hd, "how not?");
1.224 - _g1h->put_region_on_unclean_list(hd);
1.225 - if (!hd->isHumongous()) {
1.226 - // Add this to the _free_regions count by 1.
1.227 - _g1h->finish_free_region_work(0, 0, 1, NULL);
1.228 + G1CollectedHeap* g1h = G1CollectedHeap::heap();
1.229 +
1.230 + _cleanup_list.verify_optional();
1.231 + FreeRegionList local_free_list("Local Cleanup List");
1.232 +
1.233 + if (G1ConcRegionFreeingVerbose) {
1.234 + gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1.235 + "cleanup list has "SIZE_FORMAT" entries",
1.236 + _cleanup_list.length());
1.237 + }
1.238 +
1.239 + // Noone else should be accessing the _cleanup_list at this point,
1.240 + // so it's not necessary to take any locks
1.241 + while (!_cleanup_list.is_empty()) {
1.242 + HeapRegion* hr = _cleanup_list.remove_head();
1.243 + assert(hr != NULL, "the list was not empty");
1.244 + hr->rem_set()->clear();
1.245 + local_free_list.add_as_tail(hr);
1.246 +
1.247 + // Instead of adding one region at a time to the secondary_free_list,
1.248 + // we accumulate them in the local list and move them a few at a
1.249 + // time. This also cuts down on the number of notify_all() calls
1.250 + // we do during this process. We'll also append the local list when
1.251 + // _cleanup_list is empty (which means we just removed the last
1.252 + // region from the _cleanup_list).
1.253 + if ((local_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1.254 + _cleanup_list.is_empty()) {
1.255 + if (G1ConcRegionFreeingVerbose) {
1.256 + gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1.257 + "appending "SIZE_FORMAT" entries to the "
1.258 + "secondary_free_list, clean list still has "
1.259 + SIZE_FORMAT" entries",
1.260 + local_free_list.length(),
1.261 + _cleanup_list.length());
1.262 }
1.263 - hd = list->hd();
1.264 - assert(hd == next_hd, "how not?");
1.265 +
1.266 + {
1.267 + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1.268 + g1h->secondary_free_list_add_as_tail(&local_free_list);
1.269 + SecondaryFreeList_lock->notify_all();
1.270 + }
1.271 +
1.272 + if (G1StressConcRegionFreeing) {
1.273 + for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1.274 + os::sleep(Thread::current(), (jlong) 1, false);
1.275 + }
1.276 + }
1.277 }
1.278 }
1.279 + assert(local_free_list.is_empty(), "post-condition");
1.280 }
1.281
1.282 bool G1CMIsAliveClosure::do_object_b(oop obj) {
1.283 @@ -2897,9 +2922,9 @@
1.284 virtual void do_oop( oop* p) { do_oop_work(p); }
1.285
1.286 template <class T> void do_oop_work(T* p) {
1.287 - assert(_g1h->is_in_g1_reserved((HeapWord*) p), "invariant");
1.288 - assert(!_g1h->heap_region_containing((HeapWord*) p)->is_on_free_list(),
1.289 - "invariant");
1.290 + assert( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant");
1.291 + assert(!_g1h->is_on_free_list(
1.292 + _g1h->heap_region_containing((HeapWord*) p)), "invariant");
1.293
1.294 oop obj = oopDesc::load_decode_heap_oop(p);
1.295 if (_cm->verbose_high())
1.296 @@ -3119,8 +3144,8 @@
1.297 void CMTask::push(oop obj) {
1.298 HeapWord* objAddr = (HeapWord*) obj;
1.299 assert(_g1h->is_in_g1_reserved(objAddr), "invariant");
1.300 - assert(!_g1h->heap_region_containing(objAddr)->is_on_free_list(),
1.301 - "invariant");
1.302 + assert(!_g1h->is_on_free_list(
1.303 + _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant");
1.304 assert(!_g1h->is_obj_ill(obj), "invariant");
1.305 assert(_nextMarkBitMap->isMarked(objAddr), "invariant");
1.306
1.307 @@ -3365,8 +3390,8 @@
1.308 (void*) obj);
1.309
1.310 assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" );
1.311 - assert(!_g1h->heap_region_containing(obj)->is_on_free_list(),
1.312 - "invariant");
1.313 + assert(!_g1h->is_on_free_list(
1.314 + _g1h->heap_region_containing((HeapWord*) obj)), "invariant");
1.315
1.316 scan_object(obj);
1.317
