1.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Mon Jul 21 10:00:31 2014 +0200
1.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Wed Jul 23 09:03:32 2014 +0200
1.3 @@ -1928,6 +1928,8 @@
1.4 _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()),
1.5 _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()),
1.6 _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()),
1.7 + _humongous_is_live(),
1.8 + _has_humongous_reclaim_candidates(false),
1.9 _free_regions_coming(false),
1.10 _young_list(new YoungList(this)),
1.11 _gc_time_stamp(0),
1.12 @@ -2084,6 +2086,7 @@
1.13 _g1h = this;
1.14
1.15 _in_cset_fast_test.initialize(_g1_reserved.start(), _g1_reserved.end(), HeapRegion::GrainBytes);
1.16 + _humongous_is_live.initialize(_g1_reserved.start(), _g1_reserved.end(), HeapRegion::GrainBytes);
1.17
1.18 // Create the ConcurrentMark data structure and thread.
1.19 // (Must do this late, so that "max_regions" is defined.)
1.20 @@ -2179,6 +2182,11 @@
1.21 }
1.22 }
1.23
1.24 +void G1CollectedHeap::clear_humongous_is_live_table() {
1.25 + guarantee(G1ReclaimDeadHumongousObjectsAtYoungGC, "Should only be called if true");
1.26 + _humongous_is_live.clear();
1.27 +}
1.28 +
1.29 size_t G1CollectedHeap::conservative_max_heap_alignment() {
1.30 return HeapRegion::max_region_size();
1.31 }
1.32 @@ -3797,6 +3805,61 @@
1.33 return g1_rem_set()->cardsScanned();
1.34 }
1.35
1.36 +bool G1CollectedHeap::humongous_region_is_always_live(uint index) {
1.37 + HeapRegion* region = region_at(index);
1.38 + assert(region->startsHumongous(), "Must start a humongous object");
1.39 + return oop(region->bottom())->is_objArray() || !region->rem_set()->is_empty();
1.40 +}
1.41 +
1.42 +class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure {
1.43 + private:
1.44 + size_t _total_humongous;
1.45 + size_t _candidate_humongous;
1.46 + public:
1.47 + RegisterHumongousWithInCSetFastTestClosure() : _total_humongous(0), _candidate_humongous(0) {
1.48 + }
1.49 +
1.50 + virtual bool doHeapRegion(HeapRegion* r) {
1.51 + if (!r->startsHumongous()) {
1.52 + return false;
1.53 + }
1.54 + G1CollectedHeap* g1h = G1CollectedHeap::heap();
1.55 +
1.56 + uint region_idx = r->hrs_index();
1.57 + bool is_candidate = !g1h->humongous_region_is_always_live(region_idx);
1.58 + // Is_candidate already filters out humongous regions with some remembered set.
1.59 + // This will not lead to humongous object that we mistakenly keep alive because
1.60 + // during young collection the remembered sets will only be added to.
1.61 + if (is_candidate) {
1.62 + g1h->register_humongous_region_with_in_cset_fast_test(region_idx);
1.63 + _candidate_humongous++;
1.64 + }
1.65 + _total_humongous++;
1.66 +
1.67 + return false;
1.68 + }
1.69 +
1.70 + size_t total_humongous() const { return _total_humongous; }
1.71 + size_t candidate_humongous() const { return _candidate_humongous; }
1.72 +};
1.73 +
1.74 +void G1CollectedHeap::register_humongous_regions_with_in_cset_fast_test() {
1.75 + if (!G1ReclaimDeadHumongousObjectsAtYoungGC) {
1.76 + g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0, 0);
1.77 + return;
1.78 + }
1.79 +
1.80 + RegisterHumongousWithInCSetFastTestClosure cl;
1.81 + heap_region_iterate(&cl);
1.82 + g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(cl.total_humongous(),
1.83 + cl.candidate_humongous());
1.84 + _has_humongous_reclaim_candidates = cl.candidate_humongous() > 0;
1.85 +
1.86 + if (_has_humongous_reclaim_candidates) {
1.87 + clear_humongous_is_live_table();
1.88 + }
1.89 +}
1.90 +
1.91 void
1.92 G1CollectedHeap::setup_surviving_young_words() {
1.93 assert(_surviving_young_words == NULL, "pre-condition");
1.94 @@ -4083,6 +4146,8 @@
1.95
1.96 g1_policy()->finalize_cset(target_pause_time_ms, evacuation_info);
1.97
1.98 + register_humongous_regions_with_in_cset_fast_test();
1.99 +
1.100 _cm->note_start_of_gc();
1.101 // We should not verify the per-thread SATB buffers given that
1.102 // we have not filtered them yet (we'll do so during the
1.103 @@ -4133,6 +4198,9 @@
1.104 true /* verify_fingers */);
1.105
1.106 free_collection_set(g1_policy()->collection_set(), evacuation_info);
1.107 +
1.108 + eagerly_reclaim_humongous_regions();
1.109 +
1.110 g1_policy()->clear_collection_set();
1.111
1.112 cleanup_surviving_young_words();
1.113 @@ -4644,7 +4712,9 @@
1.114
1.115 assert(_worker_id == _par_scan_state->queue_num(), "sanity");
1.116
1.117 - if (_g1->in_cset_fast_test(obj)) {
1.118 + G1CollectedHeap::in_cset_state_t state = _g1->in_cset_state(obj);
1.119 +
1.120 + if (state == G1CollectedHeap::InCSet) {
1.121 oop forwardee;
1.122 if (obj->is_forwarded()) {
1.123 forwardee = obj->forwardee();
1.124 @@ -4663,6 +4733,9 @@
1.125 do_klass_barrier(p, forwardee);
1.126 }
1.127 } else {
1.128 + if (state == G1CollectedHeap::IsHumongous) {
1.129 + _g1->set_humongous_is_live(obj);
1.130 + }
1.131 // The object is not in collection set. If we're a root scanning
1.132 // closure during an initial mark pause then attempt to mark the object.
1.133 if (do_mark_object == G1MarkFromRoot) {
1.134 @@ -5492,12 +5565,21 @@
1.135 public:
1.136 G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
1.137 void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
1.138 - void do_oop( oop* p) {
1.139 + void do_oop(oop* p) {
1.140 oop obj = *p;
1.141
1.142 - if (_g1->obj_in_cs(obj)) {
1.143 + G1CollectedHeap::in_cset_state_t cset_state = _g1->in_cset_state(obj);
1.144 + if (obj == NULL || cset_state == G1CollectedHeap::InNeither) {
1.145 + return;
1.146 + }
1.147 + if (cset_state == G1CollectedHeap::InCSet) {
1.148 assert( obj->is_forwarded(), "invariant" );
1.149 *p = obj->forwardee();
1.150 + } else {
1.151 + assert(!obj->is_forwarded(), "invariant" );
1.152 + assert(cset_state == G1CollectedHeap::IsHumongous,
1.153 + err_msg("Only allowed InCSet state is IsHumongous, but is %d", cset_state));
1.154 + _g1->set_humongous_is_live(obj);
1.155 }
1.156 }
1.157 };
1.158 @@ -5527,7 +5609,7 @@
1.159 template <class T> void do_oop_work(T* p) {
1.160 oop obj = oopDesc::load_decode_heap_oop(p);
1.161
1.162 - if (_g1h->obj_in_cs(obj)) {
1.163 + if (_g1h->is_in_cset_or_humongous(obj)) {
1.164 // If the referent object has been forwarded (either copied
1.165 // to a new location or to itself in the event of an
1.166 // evacuation failure) then we need to update the reference
1.167 @@ -5552,10 +5634,10 @@
1.168 assert(!Metaspace::contains((const void*)p),
1.169 err_msg("Unexpectedly found a pointer from metadata: "
1.170 PTR_FORMAT, p));
1.171 - _copy_non_heap_obj_cl->do_oop(p);
1.172 - }
1.173 + _copy_non_heap_obj_cl->do_oop(p);
1.174 }
1.175 }
1.176 + }
1.177 };
1.178
1.179 // Serial drain queue closure. Called as the 'complete_gc'
1.180 @@ -6477,6 +6559,147 @@
1.181 policy->phase_times()->record_non_young_free_cset_time_ms(non_young_time_ms);
1.182 }
1.183
1.184 +class G1FreeHumongousRegionClosure : public HeapRegionClosure {
1.185 + private:
1.186 + FreeRegionList* _free_region_list;
1.187 + HeapRegionSet* _proxy_set;
1.188 + HeapRegionSetCount _humongous_regions_removed;
1.189 + size_t _freed_bytes;
1.190 + public:
1.191 +
1.192 + G1FreeHumongousRegionClosure(FreeRegionList* free_region_list) :
1.193 + _free_region_list(free_region_list), _humongous_regions_removed(), _freed_bytes(0) {
1.194 + }
1.195 +
1.196 + virtual bool doHeapRegion(HeapRegion* r) {
1.197 + if (!r->startsHumongous()) {
1.198 + return false;
1.199 + }
1.200 +
1.201 + G1CollectedHeap* g1h = G1CollectedHeap::heap();
1.202 +
1.203 + // The following checks whether the humongous object is live are sufficient.
1.204 + // The main additional check (in addition to having a reference from the roots
1.205 + // or the young gen) is whether the humongous object has a remembered set entry.
1.206 + //
1.207 + // A humongous object cannot be live if there is no remembered set for it
1.208 + // because:
1.209 + // - there can be no references from within humongous starts regions referencing
1.210 + // the object because we never allocate other objects into them.
1.211 + // (I.e. there are no intra-region references that may be missed by the
1.212 + // remembered set)
1.213 + // - as soon there is a remembered set entry to the humongous starts region
1.214 + // (i.e. it has "escaped" to an old object) this remembered set entry will stay
1.215 + // until the end of a concurrent mark.
1.216 + //
1.217 + // It is not required to check whether the object has been found dead by marking
1.218 + // or not, in fact it would prevent reclamation within a concurrent cycle, as
1.219 + // all objects allocated during that time are considered live.
1.220 + // SATB marking is even more conservative than the remembered set.
1.221 + // So if at this point in the collection there is no remembered set entry,
1.222 + // nobody has a reference to it.
1.223 + // At the start of collection we flush all refinement logs, and remembered sets
1.224 + // are completely up-to-date wrt to references to the humongous object.
1.225 + //
1.226 + // Other implementation considerations:
1.227 + // - never consider object arrays: while they are a valid target, they have not
1.228 + // been observed to be used as temporary objects.
1.229 + // - they would also pose considerable effort for cleaning up the the remembered
1.230 + // sets.
1.231 + // While this cleanup is not strictly necessary to be done (or done instantly),
1.232 + // given that their occurrence is very low, this saves us this additional
1.233 + // complexity.
1.234 + uint region_idx = r->hrs_index();
1.235 + if (g1h->humongous_is_live(region_idx) ||
1.236 + g1h->humongous_region_is_always_live(region_idx)) {
1.237 +
1.238 + if (G1TraceReclaimDeadHumongousObjectsAtYoungGC) {
1.239 + gclog_or_tty->print_cr("Live humongous %d region %d with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is dead-bitmap %d live-other %d obj array %d",
1.240 + r->isHumongous(),
1.241 + region_idx,
1.242 + r->rem_set()->occupied(),
1.243 + r->rem_set()->strong_code_roots_list_length(),
1.244 + g1h->mark_in_progress() && !g1h->g1_policy()->during_initial_mark_pause(),
1.245 + g1h->humongous_is_live(region_idx),
1.246 + oop(r->bottom())->is_objArray()
1.247 + );
1.248 + }
1.249 +
1.250 + return false;
1.251 + }
1.252 +
1.253 + guarantee(!((oop)(r->bottom()))->is_objArray(),
1.254 + err_msg("Eagerly reclaiming object arrays is not supported, but the object "PTR_FORMAT" is.",
1.255 + r->bottom()));
1.256 +
1.257 + if (G1TraceReclaimDeadHumongousObjectsAtYoungGC) {
1.258 + gclog_or_tty->print_cr("Reclaim humongous region %d start "PTR_FORMAT" region %d length "UINT32_FORMAT" with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is dead-bitmap %d live-other %d obj array %d",
1.259 + r->isHumongous(),
1.260 + r->bottom(),
1.261 + region_idx,
1.262 + r->region_num(),
1.263 + r->rem_set()->occupied(),
1.264 + r->rem_set()->strong_code_roots_list_length(),
1.265 + g1h->mark_in_progress() && !g1h->g1_policy()->during_initial_mark_pause(),
1.266 + g1h->humongous_is_live(region_idx),
1.267 + oop(r->bottom())->is_objArray()
1.268 + );
1.269 + }
1.270 + _freed_bytes += r->used();
1.271 + r->set_containing_set(NULL);
1.272 + _humongous_regions_removed.increment(1u, r->capacity());
1.273 + g1h->free_humongous_region(r, _free_region_list, false);
1.274 +
1.275 + return false;
1.276 + }
1.277 +
1.278 + HeapRegionSetCount& humongous_free_count() {
1.279 + return _humongous_regions_removed;
1.280 + }
1.281 +
1.282 + size_t bytes_freed() const {
1.283 + return _freed_bytes;
1.284 + }
1.285 +
1.286 + size_t humongous_reclaimed() const {
1.287 + return _humongous_regions_removed.length();
1.288 + }
1.289 +};
1.290 +
1.291 +void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
1.292 + assert_at_safepoint(true);
1.293 +
1.294 + if (!G1ReclaimDeadHumongousObjectsAtYoungGC || !_has_humongous_reclaim_candidates) {
1.295 + g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms(0.0, 0);
1.296 + return;
1.297 + }
1.298 +
1.299 + double start_time = os::elapsedTime();
1.300 +
1.301 + FreeRegionList local_cleanup_list("Local Humongous Cleanup List");
1.302 +
1.303 + G1FreeHumongousRegionClosure cl(&local_cleanup_list);
1.304 + heap_region_iterate(&cl);
1.305 +
1.306 + HeapRegionSetCount empty_set;
1.307 + remove_from_old_sets(empty_set, cl.humongous_free_count());
1.308 +
1.309 + G1HRPrinter* hr_printer = _g1h->hr_printer();
1.310 + if (hr_printer->is_active()) {
1.311 + FreeRegionListIterator iter(&local_cleanup_list);
1.312 + while (iter.more_available()) {
1.313 + HeapRegion* hr = iter.get_next();
1.314 + hr_printer->cleanup(hr);
1.315 + }
1.316 + }
1.317 +
1.318 + prepend_to_freelist(&local_cleanup_list);
1.319 + decrement_summary_bytes(cl.bytes_freed());
1.320 +
1.321 + g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms((os::elapsedTime() - start_time) * 1000.0,
1.322 + cl.humongous_reclaimed());
1.323 +}
1.324 +
1.325 // This routine is similar to the above but does not record
1.326 // any policy statistics or update free lists; we are abandoning
1.327 // the current incremental collection set in preparation of a
