1.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Mon Jul 21 10:00:31 2014 +0200 1.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Wed Jul 23 09:03:32 2014 +0200 1.3 @@ -1914,6 +1914,8 @@ 1.4 _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()), 1.5 _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()), 1.6 _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()), 1.7 + _humongous_is_live(), 1.8 + _has_humongous_reclaim_candidates(false), 1.9 _free_regions_coming(false), 1.10 _young_list(new YoungList(this)), 1.11 _gc_time_stamp(0), 1.12 @@ -2070,6 +2072,7 @@ 1.13 _g1h = this; 1.14 1.15 _in_cset_fast_test.initialize(_g1_reserved.start(), _g1_reserved.end(), HeapRegion::GrainBytes); 1.16 + _humongous_is_live.initialize(_g1_reserved.start(), _g1_reserved.end(), HeapRegion::GrainBytes); 1.17 1.18 // Create the ConcurrentMark data structure and thread. 1.19 // (Must do this late, so that "max_regions" is defined.) 1.20 @@ -2165,6 +2168,11 @@ 1.21 } 1.22 } 1.23 1.24 +void G1CollectedHeap::clear_humongous_is_live_table() { 1.25 + guarantee(G1ReclaimDeadHumongousObjectsAtYoungGC, "Should only be called if true"); 1.26 + _humongous_is_live.clear(); 1.27 +} 1.28 + 1.29 size_t G1CollectedHeap::conservative_max_heap_alignment() { 1.30 return HeapRegion::max_region_size(); 1.31 } 1.32 @@ -3783,6 +3791,61 @@ 1.33 return g1_rem_set()->cardsScanned(); 1.34 } 1.35 1.36 +bool G1CollectedHeap::humongous_region_is_always_live(uint index) { 1.37 + HeapRegion* region = region_at(index); 1.38 + assert(region->startsHumongous(), "Must start a humongous object"); 1.39 + return oop(region->bottom())->is_objArray() || !region->rem_set()->is_empty(); 1.40 +} 1.41 + 1.42 +class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure { 1.43 + private: 1.44 + size_t _total_humongous; 1.45 + size_t _candidate_humongous; 1.46 + public: 1.47 + RegisterHumongousWithInCSetFastTestClosure() : _total_humongous(0), _candidate_humongous(0) { 1.48 + } 1.49 + 1.50 + virtual bool doHeapRegion(HeapRegion* r) { 1.51 + if (!r->startsHumongous()) { 1.52 + return false; 1.53 + } 1.54 + G1CollectedHeap* g1h = G1CollectedHeap::heap(); 1.55 + 1.56 + uint region_idx = r->hrs_index(); 1.57 + bool is_candidate = !g1h->humongous_region_is_always_live(region_idx); 1.58 + // Is_candidate already filters out humongous regions with some remembered set. 1.59 + // This will not lead to humongous object that we mistakenly keep alive because 1.60 + // during young collection the remembered sets will only be added to. 1.61 + if (is_candidate) { 1.62 + g1h->register_humongous_region_with_in_cset_fast_test(region_idx); 1.63 + _candidate_humongous++; 1.64 + } 1.65 + _total_humongous++; 1.66 + 1.67 + return false; 1.68 + } 1.69 + 1.70 + size_t total_humongous() const { return _total_humongous; } 1.71 + size_t candidate_humongous() const { return _candidate_humongous; } 1.72 +}; 1.73 + 1.74 +void G1CollectedHeap::register_humongous_regions_with_in_cset_fast_test() { 1.75 + if (!G1ReclaimDeadHumongousObjectsAtYoungGC) { 1.76 + g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0, 0); 1.77 + return; 1.78 + } 1.79 + 1.80 + RegisterHumongousWithInCSetFastTestClosure cl; 1.81 + heap_region_iterate(&cl); 1.82 + g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(cl.total_humongous(), 1.83 + cl.candidate_humongous()); 1.84 + _has_humongous_reclaim_candidates = cl.candidate_humongous() > 0; 1.85 + 1.86 + if (_has_humongous_reclaim_candidates) { 1.87 + clear_humongous_is_live_table(); 1.88 + } 1.89 +} 1.90 + 1.91 void 1.92 G1CollectedHeap::setup_surviving_young_words() { 1.93 assert(_surviving_young_words == NULL, "pre-condition"); 1.94 @@ -4068,6 +4131,8 @@ 1.95 1.96 g1_policy()->finalize_cset(target_pause_time_ms, evacuation_info); 1.97 1.98 + register_humongous_regions_with_in_cset_fast_test(); 1.99 + 1.100 _cm->note_start_of_gc(); 1.101 // We should not verify the per-thread SATB buffers given that 1.102 // we have not filtered them yet (we'll do so during the 1.103 @@ -4118,6 +4183,9 @@ 1.104 true /* verify_fingers */); 1.105 1.106 free_collection_set(g1_policy()->collection_set(), evacuation_info); 1.107 + 1.108 + eagerly_reclaim_humongous_regions(); 1.109 + 1.110 g1_policy()->clear_collection_set(); 1.111 1.112 cleanup_surviving_young_words(); 1.113 @@ -4628,7 +4696,9 @@ 1.114 1.115 assert(_worker_id == _par_scan_state->queue_num(), "sanity"); 1.116 1.117 - if (_g1->in_cset_fast_test(obj)) { 1.118 + G1CollectedHeap::in_cset_state_t state = _g1->in_cset_state(obj); 1.119 + 1.120 + if (state == G1CollectedHeap::InCSet) { 1.121 oop forwardee; 1.122 if (obj->is_forwarded()) { 1.123 forwardee = obj->forwardee(); 1.124 @@ -4647,6 +4717,9 @@ 1.125 do_klass_barrier(p, forwardee); 1.126 } 1.127 } else { 1.128 + if (state == G1CollectedHeap::IsHumongous) { 1.129 + _g1->set_humongous_is_live(obj); 1.130 + } 1.131 // The object is not in collection set. If we're a root scanning 1.132 // closure during an initial mark pause then attempt to mark the object. 1.133 if (do_mark_object == G1MarkFromRoot) { 1.134 @@ -5476,12 +5549,21 @@ 1.135 public: 1.136 G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} 1.137 void do_oop(narrowOop* p) { guarantee(false, "Not needed"); } 1.138 - void do_oop( oop* p) { 1.139 + void do_oop(oop* p) { 1.140 oop obj = *p; 1.141 1.142 - if (_g1->obj_in_cs(obj)) { 1.143 + G1CollectedHeap::in_cset_state_t cset_state = _g1->in_cset_state(obj); 1.144 + if (obj == NULL || cset_state == G1CollectedHeap::InNeither) { 1.145 + return; 1.146 + } 1.147 + if (cset_state == G1CollectedHeap::InCSet) { 1.148 assert( obj->is_forwarded(), "invariant" ); 1.149 *p = obj->forwardee(); 1.150 + } else { 1.151 + assert(!obj->is_forwarded(), "invariant" ); 1.152 + assert(cset_state == G1CollectedHeap::IsHumongous, 1.153 + err_msg("Only allowed InCSet state is IsHumongous, but is %d", cset_state)); 1.154 + _g1->set_humongous_is_live(obj); 1.155 } 1.156 } 1.157 }; 1.158 @@ -5511,7 +5593,7 @@ 1.159 template <class T> void do_oop_work(T* p) { 1.160 oop obj = oopDesc::load_decode_heap_oop(p); 1.161 1.162 - if (_g1h->obj_in_cs(obj)) { 1.163 + if (_g1h->is_in_cset_or_humongous(obj)) { 1.164 // If the referent object has been forwarded (either copied 1.165 // to a new location or to itself in the event of an 1.166 // evacuation failure) then we need to update the reference 1.167 @@ -5536,10 +5618,10 @@ 1.168 assert(!Metaspace::contains((const void*)p), 1.169 err_msg("Unexpectedly found a pointer from metadata: " 1.170 PTR_FORMAT, p)); 1.171 - _copy_non_heap_obj_cl->do_oop(p); 1.172 - } 1.173 + _copy_non_heap_obj_cl->do_oop(p); 1.174 } 1.175 } 1.176 + } 1.177 }; 1.178 1.179 // Serial drain queue closure. Called as the 'complete_gc' 1.180 @@ -6376,6 +6458,147 @@ 1.181 policy->phase_times()->record_non_young_free_cset_time_ms(non_young_time_ms); 1.182 } 1.183 1.184 +class G1FreeHumongousRegionClosure : public HeapRegionClosure { 1.185 + private: 1.186 + FreeRegionList* _free_region_list; 1.187 + HeapRegionSet* _proxy_set; 1.188 + HeapRegionSetCount _humongous_regions_removed; 1.189 + size_t _freed_bytes; 1.190 + public: 1.191 + 1.192 + G1FreeHumongousRegionClosure(FreeRegionList* free_region_list) : 1.193 + _free_region_list(free_region_list), _humongous_regions_removed(), _freed_bytes(0) { 1.194 + } 1.195 + 1.196 + virtual bool doHeapRegion(HeapRegion* r) { 1.197 + if (!r->startsHumongous()) { 1.198 + return false; 1.199 + } 1.200 + 1.201 + G1CollectedHeap* g1h = G1CollectedHeap::heap(); 1.202 + 1.203 + // The following checks whether the humongous object is live are sufficient. 1.204 + // The main additional check (in addition to having a reference from the roots 1.205 + // or the young gen) is whether the humongous object has a remembered set entry. 1.206 + // 1.207 + // A humongous object cannot be live if there is no remembered set for it 1.208 + // because: 1.209 + // - there can be no references from within humongous starts regions referencing 1.210 + // the object because we never allocate other objects into them. 1.211 + // (I.e. there are no intra-region references that may be missed by the 1.212 + // remembered set) 1.213 + // - as soon there is a remembered set entry to the humongous starts region 1.214 + // (i.e. it has "escaped" to an old object) this remembered set entry will stay 1.215 + // until the end of a concurrent mark. 1.216 + // 1.217 + // It is not required to check whether the object has been found dead by marking 1.218 + // or not, in fact it would prevent reclamation within a concurrent cycle, as 1.219 + // all objects allocated during that time are considered live. 1.220 + // SATB marking is even more conservative than the remembered set. 1.221 + // So if at this point in the collection there is no remembered set entry, 1.222 + // nobody has a reference to it. 1.223 + // At the start of collection we flush all refinement logs, and remembered sets 1.224 + // are completely up-to-date wrt to references to the humongous object. 1.225 + // 1.226 + // Other implementation considerations: 1.227 + // - never consider object arrays: while they are a valid target, they have not 1.228 + // been observed to be used as temporary objects. 1.229 + // - they would also pose considerable effort for cleaning up the the remembered 1.230 + // sets. 1.231 + // While this cleanup is not strictly necessary to be done (or done instantly), 1.232 + // given that their occurrence is very low, this saves us this additional 1.233 + // complexity. 1.234 + uint region_idx = r->hrs_index(); 1.235 + if (g1h->humongous_is_live(region_idx) || 1.236 + g1h->humongous_region_is_always_live(region_idx)) { 1.237 + 1.238 + if (G1TraceReclaimDeadHumongousObjectsAtYoungGC) { 1.239 + gclog_or_tty->print_cr("Live humongous %d region %d with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is dead-bitmap %d live-other %d obj array %d", 1.240 + r->isHumongous(), 1.241 + region_idx, 1.242 + r->rem_set()->occupied(), 1.243 + r->rem_set()->strong_code_roots_list_length(), 1.244 + g1h->mark_in_progress() && !g1h->g1_policy()->during_initial_mark_pause(), 1.245 + g1h->humongous_is_live(region_idx), 1.246 + oop(r->bottom())->is_objArray() 1.247 + ); 1.248 + } 1.249 + 1.250 + return false; 1.251 + } 1.252 + 1.253 + guarantee(!((oop)(r->bottom()))->is_objArray(), 1.254 + err_msg("Eagerly reclaiming object arrays is not supported, but the object "PTR_FORMAT" is.", 1.255 + r->bottom())); 1.256 + 1.257 + if (G1TraceReclaimDeadHumongousObjectsAtYoungGC) { 1.258 + gclog_or_tty->print_cr("Reclaim humongous region %d start "PTR_FORMAT" region %d length "UINT32_FORMAT" with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is dead-bitmap %d live-other %d obj array %d", 1.259 + r->isHumongous(), 1.260 + r->bottom(), 1.261 + region_idx, 1.262 + r->region_num(), 1.263 + r->rem_set()->occupied(), 1.264 + r->rem_set()->strong_code_roots_list_length(), 1.265 + g1h->mark_in_progress() && !g1h->g1_policy()->during_initial_mark_pause(), 1.266 + g1h->humongous_is_live(region_idx), 1.267 + oop(r->bottom())->is_objArray() 1.268 + ); 1.269 + } 1.270 + _freed_bytes += r->used(); 1.271 + r->set_containing_set(NULL); 1.272 + _humongous_regions_removed.increment(1u, r->capacity()); 1.273 + g1h->free_humongous_region(r, _free_region_list, false); 1.274 + 1.275 + return false; 1.276 + } 1.277 + 1.278 + HeapRegionSetCount& humongous_free_count() { 1.279 + return _humongous_regions_removed; 1.280 + } 1.281 + 1.282 + size_t bytes_freed() const { 1.283 + return _freed_bytes; 1.284 + } 1.285 + 1.286 + size_t humongous_reclaimed() const { 1.287 + return _humongous_regions_removed.length(); 1.288 + } 1.289 +}; 1.290 + 1.291 +void G1CollectedHeap::eagerly_reclaim_humongous_regions() { 1.292 + assert_at_safepoint(true); 1.293 + 1.294 + if (!G1ReclaimDeadHumongousObjectsAtYoungGC || !_has_humongous_reclaim_candidates) { 1.295 + g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms(0.0, 0); 1.296 + return; 1.297 + } 1.298 + 1.299 + double start_time = os::elapsedTime(); 1.300 + 1.301 + FreeRegionList local_cleanup_list("Local Humongous Cleanup List"); 1.302 + 1.303 + G1FreeHumongousRegionClosure cl(&local_cleanup_list); 1.304 + heap_region_iterate(&cl); 1.305 + 1.306 + HeapRegionSetCount empty_set; 1.307 + remove_from_old_sets(empty_set, cl.humongous_free_count()); 1.308 + 1.309 + G1HRPrinter* hr_printer = _g1h->hr_printer(); 1.310 + if (hr_printer->is_active()) { 1.311 + FreeRegionListIterator iter(&local_cleanup_list); 1.312 + while (iter.more_available()) { 1.313 + HeapRegion* hr = iter.get_next(); 1.314 + hr_printer->cleanup(hr); 1.315 + } 1.316 + } 1.317 + 1.318 + prepend_to_freelist(&local_cleanup_list); 1.319 + decrement_summary_bytes(cl.bytes_freed()); 1.320 + 1.321 + g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms((os::elapsedTime() - start_time) * 1000.0, 1.322 + cl.humongous_reclaimed()); 1.323 +} 1.324 + 1.325 // This routine is similar to the above but does not record 1.326 // any policy statistics or update free lists; we are abandoning 1.327 // the current incremental collection set in preparation of a