Wed, 07 Jan 2015 15:15:37 +0100
8048179: Early reclaim of large objects that are referenced by a few objects
Summary: Push the remembered sets of large objects with few referenced into the dirty card queue at the beginning of the evacuation so that they may end up with zero remembered set entries at the end of the collection, and are potentially reclaimed. Also improve timing measurements of the early reclaim mechanism, and shorten flag names.
Reviewed-by: brutisso, jmasa, dfazunen
1 /*
2 * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
25 #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONREMSET_HPP
26 #define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONREMSET_HPP
28 #include "gc_implementation/g1/g1CodeCacheRemSet.hpp"
29 #include "gc_implementation/g1/sparsePRT.hpp"
31 // Remembered set for a heap region. Represent a set of "cards" that
32 // contain pointers into the owner heap region. Cards are defined somewhat
33 // abstractly, in terms of what the "BlockOffsetTable" in use can parse.
35 class G1CollectedHeap;
36 class G1BlockOffsetSharedArray;
37 class HeapRegion;
38 class HeapRegionRemSetIterator;
39 class PerRegionTable;
40 class SparsePRT;
41 class nmethod;
43 // Essentially a wrapper around SparsePRTCleanupTask. See
44 // sparsePRT.hpp for more details.
45 class HRRSCleanupTask : public SparsePRTCleanupTask {
46 };
48 // The FromCardCache remembers the most recently processed card on the heap on
49 // a per-region and per-thread basis.
50 class FromCardCache : public AllStatic {
51 private:
52 // Array of card indices. Indexed by thread X and heap region to minimize
53 // thread contention.
54 static int** _cache;
55 static uint _max_regions;
56 static size_t _static_mem_size;
58 public:
59 enum {
60 InvalidCard = -1 // Card value of an invalid card, i.e. a card index not otherwise used.
61 };
63 static void clear(uint region_idx);
65 // Returns true if the given card is in the cache at the given location, or
66 // replaces the card at that location and returns false.
67 static bool contains_or_replace(uint worker_id, uint region_idx, int card) {
68 int card_in_cache = at(worker_id, region_idx);
69 if (card_in_cache == card) {
70 return true;
71 } else {
72 set(worker_id, region_idx, card);
73 return false;
74 }
75 }
77 static int at(uint worker_id, uint region_idx) {
78 return _cache[worker_id][region_idx];
79 }
81 static void set(uint worker_id, uint region_idx, int val) {
82 _cache[worker_id][region_idx] = val;
83 }
85 static void initialize(uint n_par_rs, uint max_num_regions);
87 static void invalidate(uint start_idx, size_t num_regions);
89 static void print(outputStream* out = gclog_or_tty) PRODUCT_RETURN;
91 static size_t static_mem_size() {
92 return _static_mem_size;
93 }
94 };
96 // The "_coarse_map" is a bitmap with one bit for each region, where set
97 // bits indicate that the corresponding region may contain some pointer
98 // into the owning region.
100 // The "_fine_grain_entries" array is an open hash table of PerRegionTables
101 // (PRTs), indicating regions for which we're keeping the RS as a set of
102 // cards. The strategy is to cap the size of the fine-grain table,
103 // deleting an entry and setting the corresponding coarse-grained bit when
104 // we would overflow this cap.
106 // We use a mixture of locking and lock-free techniques here. We allow
107 // threads to locate PRTs without locking, but threads attempting to alter
108 // a bucket list obtain a lock. This means that any failing attempt to
109 // find a PRT must be retried with the lock. It might seem dangerous that
110 // a read can find a PRT that is concurrently deleted. This is all right,
111 // because:
112 //
113 // 1) We only actually free PRT's at safe points (though we reuse them at
114 // other times).
115 // 2) We find PRT's in an attempt to add entries. If a PRT is deleted,
116 // it's _coarse_map bit is set, so the that we were attempting to add
117 // is represented. If a deleted PRT is re-used, a thread adding a bit,
118 // thinking the PRT is for a different region, does no harm.
120 class OtherRegionsTable VALUE_OBJ_CLASS_SPEC {
121 friend class HeapRegionRemSetIterator;
123 G1CollectedHeap* _g1h;
124 Mutex* _m;
125 HeapRegion* _hr;
127 // These are protected by "_m".
128 BitMap _coarse_map;
129 size_t _n_coarse_entries;
130 static jint _n_coarsenings;
132 PerRegionTable** _fine_grain_regions;
133 size_t _n_fine_entries;
135 // The fine grain remembered sets are doubly linked together using
136 // their 'next' and 'prev' fields.
137 // This allows fast bulk freeing of all the fine grain remembered
138 // set entries, and fast finding of all of them without iterating
139 // over the _fine_grain_regions table.
140 PerRegionTable * _first_all_fine_prts;
141 PerRegionTable * _last_all_fine_prts;
143 // Used to sample a subset of the fine grain PRTs to determine which
144 // PRT to evict and coarsen.
145 size_t _fine_eviction_start;
146 static size_t _fine_eviction_stride;
147 static size_t _fine_eviction_sample_size;
149 SparsePRT _sparse_table;
151 // These are static after init.
152 static size_t _max_fine_entries;
153 static size_t _mod_max_fine_entries_mask;
155 // Requires "prt" to be the first element of the bucket list appropriate
156 // for "hr". If this list contains an entry for "hr", return it,
157 // otherwise return "NULL".
158 PerRegionTable* find_region_table(size_t ind, HeapRegion* hr) const;
160 // Find, delete, and return a candidate PerRegionTable, if any exists,
161 // adding the deleted region to the coarse bitmap. Requires the caller
162 // to hold _m, and the fine-grain table to be full.
163 PerRegionTable* delete_region_table();
165 // If a PRT for "hr" is in the bucket list indicated by "ind" (which must
166 // be the correct index for "hr"), delete it and return true; else return
167 // false.
168 bool del_single_region_table(size_t ind, HeapRegion* hr);
170 // link/add the given fine grain remembered set into the "all" list
171 void link_to_all(PerRegionTable * prt);
172 // unlink/remove the given fine grain remembered set into the "all" list
173 void unlink_from_all(PerRegionTable * prt);
175 public:
176 OtherRegionsTable(HeapRegion* hr, Mutex* m);
178 HeapRegion* hr() const { return _hr; }
180 // For now. Could "expand" some tables in the future, so that this made
181 // sense.
182 void add_reference(OopOrNarrowOopStar from, int tid);
184 // Returns whether this remembered set (and all sub-sets) have an occupancy
185 // that is less or equal than the given occupancy.
186 bool occupancy_less_or_equal_than(size_t limit) const;
188 // Removes any entries shown by the given bitmaps to contain only dead
189 // objects.
190 void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm);
192 // Returns whether this remembered set (and all sub-sets) contain no entries.
193 bool is_empty() const;
195 size_t occupied() const;
196 size_t occ_fine() const;
197 size_t occ_coarse() const;
198 size_t occ_sparse() const;
200 static jint n_coarsenings() { return _n_coarsenings; }
202 // Returns size in bytes.
203 // Not const because it takes a lock.
204 size_t mem_size() const;
205 static size_t static_mem_size();
206 static size_t fl_mem_size();
208 bool contains_reference(OopOrNarrowOopStar from) const;
209 bool contains_reference_locked(OopOrNarrowOopStar from) const;
211 void clear();
213 // Specifically clear the from_card_cache.
214 void clear_fcc();
216 void do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task);
218 // Declare the heap size (in # of regions) to the OtherRegionsTable.
219 // (Uses it to initialize from_card_cache).
220 static void initialize(uint max_regions);
222 // Declares that regions between start_idx <= i < start_idx + num_regions are
223 // not in use. Make sure that any entries for these regions are invalid.
224 static void invalidate(uint start_idx, size_t num_regions);
226 static void print_from_card_cache();
227 };
229 class HeapRegionRemSet : public CHeapObj<mtGC> {
230 friend class VMStructs;
231 friend class HeapRegionRemSetIterator;
233 public:
234 enum Event {
235 Event_EvacStart, Event_EvacEnd, Event_RSUpdateEnd
236 };
238 private:
239 G1BlockOffsetSharedArray* _bosa;
240 G1BlockOffsetSharedArray* bosa() const { return _bosa; }
242 // A set of code blobs (nmethods) whose code contains pointers into
243 // the region that owns this RSet.
244 G1CodeRootSet _code_roots;
246 Mutex _m;
248 OtherRegionsTable _other_regions;
250 enum ParIterState { Unclaimed, Claimed, Complete };
251 volatile ParIterState _iter_state;
252 volatile jlong _iter_claimed;
254 // Unused unless G1RecordHRRSOops is true.
256 static const int MaxRecorded = 1000000;
257 static OopOrNarrowOopStar* _recorded_oops;
258 static HeapWord** _recorded_cards;
259 static HeapRegion** _recorded_regions;
260 static int _n_recorded;
262 static const int MaxRecordedEvents = 1000;
263 static Event* _recorded_events;
264 static int* _recorded_event_index;
265 static int _n_recorded_events;
267 static void print_event(outputStream* str, Event evnt);
269 public:
270 HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, HeapRegion* hr);
272 static uint num_par_rem_sets();
273 static void setup_remset_size();
275 HeapRegion* hr() const {
276 return _other_regions.hr();
277 }
279 bool is_empty() const {
280 return (strong_code_roots_list_length() == 0) && _other_regions.is_empty();
281 }
283 bool occupancy_less_or_equal_than(size_t occ) const {
284 return (strong_code_roots_list_length() == 0) && _other_regions.occupancy_less_or_equal_than(occ);
285 }
287 size_t occupied() {
288 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);
289 return occupied_locked();
290 }
291 size_t occupied_locked() {
292 return _other_regions.occupied();
293 }
294 size_t occ_fine() const {
295 return _other_regions.occ_fine();
296 }
297 size_t occ_coarse() const {
298 return _other_regions.occ_coarse();
299 }
300 size_t occ_sparse() const {
301 return _other_regions.occ_sparse();
302 }
304 static jint n_coarsenings() { return OtherRegionsTable::n_coarsenings(); }
306 // Used in the sequential case.
307 void add_reference(OopOrNarrowOopStar from) {
308 _other_regions.add_reference(from, 0);
309 }
311 // Used in the parallel case.
312 void add_reference(OopOrNarrowOopStar from, int tid) {
313 _other_regions.add_reference(from, tid);
314 }
316 // Removes any entries shown by the given bitmaps to contain only dead
317 // objects.
318 void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm);
320 // The region is being reclaimed; clear its remset, and any mention of
321 // entries for this region in other remsets.
322 void clear();
323 void clear_locked();
325 // Attempt to claim the region. Returns true iff this call caused an
326 // atomic transition from Unclaimed to Claimed.
327 bool claim_iter();
328 // Sets the iteration state to "complete".
329 void set_iter_complete();
330 // Returns "true" iff the region's iteration is complete.
331 bool iter_is_complete();
333 // Support for claiming blocks of cards during iteration
334 size_t iter_claimed() const { return (size_t)_iter_claimed; }
335 // Claim the next block of cards
336 size_t iter_claimed_next(size_t step) {
337 size_t current, next;
338 do {
339 current = iter_claimed();
340 next = current + step;
341 } while (Atomic::cmpxchg((jlong)next, &_iter_claimed, (jlong)current) != (jlong)current);
342 return current;
343 }
344 void reset_for_par_iteration();
346 bool verify_ready_for_par_iteration() {
347 return (_iter_state == Unclaimed) && (_iter_claimed == 0);
348 }
350 // The actual # of bytes this hr_remset takes up.
351 // Note also includes the strong code root set.
352 size_t mem_size() {
353 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);
354 return _other_regions.mem_size()
355 // This correction is necessary because the above includes the second
356 // part.
357 + (sizeof(HeapRegionRemSet) - sizeof(OtherRegionsTable))
358 + strong_code_roots_mem_size();
359 }
361 // Returns the memory occupancy of all static data structures associated
362 // with remembered sets.
363 static size_t static_mem_size() {
364 return OtherRegionsTable::static_mem_size() + G1CodeRootSet::static_mem_size();
365 }
367 // Returns the memory occupancy of all free_list data structures associated
368 // with remembered sets.
369 static size_t fl_mem_size() {
370 return OtherRegionsTable::fl_mem_size();
371 }
373 bool contains_reference(OopOrNarrowOopStar from) const {
374 return _other_regions.contains_reference(from);
375 }
377 // Routines for managing the list of code roots that point into
378 // the heap region that owns this RSet.
379 void add_strong_code_root(nmethod* nm);
380 void add_strong_code_root_locked(nmethod* nm);
381 void remove_strong_code_root(nmethod* nm);
383 // Applies blk->do_code_blob() to each of the entries in
384 // the strong code roots list
385 void strong_code_roots_do(CodeBlobClosure* blk) const;
387 void clean_strong_code_roots(HeapRegion* hr);
389 // Returns the number of elements in the strong code roots list
390 size_t strong_code_roots_list_length() const {
391 return _code_roots.length();
392 }
394 // Returns true if the strong code roots contains the given
395 // nmethod.
396 bool strong_code_roots_list_contains(nmethod* nm) {
397 return _code_roots.contains(nm);
398 }
400 // Returns the amount of memory, in bytes, currently
401 // consumed by the strong code roots.
402 size_t strong_code_roots_mem_size();
404 void print() PRODUCT_RETURN;
406 // Called during a stop-world phase to perform any deferred cleanups.
407 static void cleanup();
409 // Declare the heap size (in # of regions) to the HeapRegionRemSet(s).
410 // (Uses it to initialize from_card_cache).
411 static void init_heap(uint max_regions) {
412 OtherRegionsTable::initialize(max_regions);
413 }
415 static void invalidate(uint start_idx, uint num_regions) {
416 OtherRegionsTable::invalidate(start_idx, num_regions);
417 }
419 #ifndef PRODUCT
420 static void print_from_card_cache() {
421 OtherRegionsTable::print_from_card_cache();
422 }
423 #endif
425 static void record(HeapRegion* hr, OopOrNarrowOopStar f);
426 static void print_recorded();
427 static void record_event(Event evnt);
429 // These are wrappers for the similarly-named methods on
430 // SparsePRT. Look at sparsePRT.hpp for more details.
431 static void reset_for_cleanup_tasks();
432 void do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task);
433 static void finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task);
435 // Run unit tests.
436 #ifndef PRODUCT
437 static void test_prt();
438 static void test();
439 #endif
440 };
442 class HeapRegionRemSetIterator : public StackObj {
443 private:
444 // The region RSet over which we are iterating.
445 HeapRegionRemSet* _hrrs;
447 // Local caching of HRRS fields.
448 const BitMap* _coarse_map;
450 G1BlockOffsetSharedArray* _bosa;
451 G1CollectedHeap* _g1h;
453 // The number of cards yielded since initialization.
454 size_t _n_yielded_fine;
455 size_t _n_yielded_coarse;
456 size_t _n_yielded_sparse;
458 // Indicates what granularity of table that we are currently iterating over.
459 // We start iterating over the sparse table, progress to the fine grain
460 // table, and then finish with the coarse table.
461 enum IterState {
462 Sparse,
463 Fine,
464 Coarse
465 };
466 IterState _is;
468 // For both Coarse and Fine remembered set iteration this contains the
469 // first card number of the heap region we currently iterate over.
470 size_t _cur_region_card_offset;
472 // Current region index for the Coarse remembered set iteration.
473 int _coarse_cur_region_index;
474 size_t _coarse_cur_region_cur_card;
476 bool coarse_has_next(size_t& card_index);
478 // The PRT we are currently iterating over.
479 PerRegionTable* _fine_cur_prt;
480 // Card offset within the current PRT.
481 size_t _cur_card_in_prt;
483 // Update internal variables when switching to the given PRT.
484 void switch_to_prt(PerRegionTable* prt);
485 bool fine_has_next();
486 bool fine_has_next(size_t& card_index);
488 // The Sparse remembered set iterator.
489 SparsePRTIter _sparse_iter;
491 public:
492 HeapRegionRemSetIterator(HeapRegionRemSet* hrrs);
494 // If there remains one or more cards to be yielded, returns true and
495 // sets "card_index" to one of those cards (which is then considered
496 // yielded.) Otherwise, returns false (and leaves "card_index"
497 // undefined.)
498 bool has_next(size_t& card_index);
500 size_t n_yielded_fine() { return _n_yielded_fine; }
501 size_t n_yielded_coarse() { return _n_yielded_coarse; }
502 size_t n_yielded_sparse() { return _n_yielded_sparse; }
503 size_t n_yielded() {
504 return n_yielded_fine() + n_yielded_coarse() + n_yielded_sparse();
505 }
506 };
508 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONREMSET_HPP