Thu, 14 Jun 2018 09:15:08 -0700
8081202: Hotspot compile warning: "Invalid suffix on literal; C++11 requires a space between literal and identifier"
Summary: Need to add a space between macro identifier and string literal
Reviewed-by: bpittore, stefank, dholmes, kbarrett
1 /*
2 * Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
25 #include "precompiled.hpp"
26 #include "classfile/altHashing.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
29 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
30 #include "gc_implementation/g1/g1StringDedupTable.hpp"
31 #include "gc_implementation/shared/concurrentGCThread.hpp"
32 #include "memory/gcLocker.hpp"
33 #include "memory/padded.inline.hpp"
34 #include "oops/typeArrayOop.hpp"
35 #include "runtime/mutexLocker.hpp"
37 //
38 // List of deduplication table entries. Links table
39 // entries together using their _next fields.
40 //
41 class G1StringDedupEntryList : public CHeapObj<mtGC> {
42 private:
43 G1StringDedupEntry* _list;
44 size_t _length;
46 public:
47 G1StringDedupEntryList() :
48 _list(NULL),
49 _length(0) {
50 }
52 void add(G1StringDedupEntry* entry) {
53 entry->set_next(_list);
54 _list = entry;
55 _length++;
56 }
58 G1StringDedupEntry* remove() {
59 G1StringDedupEntry* entry = _list;
60 if (entry != NULL) {
61 _list = entry->next();
62 _length--;
63 }
64 return entry;
65 }
67 G1StringDedupEntry* remove_all() {
68 G1StringDedupEntry* list = _list;
69 _list = NULL;
70 return list;
71 }
73 size_t length() {
74 return _length;
75 }
76 };
78 //
79 // Cache of deduplication table entries. This cache provides fast allocation and
80 // reuse of table entries to lower the pressure on the underlying allocator.
81 // But more importantly, it provides fast/deferred freeing of table entries. This
82 // is important because freeing of table entries is done during stop-the-world
83 // phases and it is not uncommon for large number of entries to be freed at once.
84 // Tables entries that are freed during these phases are placed onto a freelist in
85 // the cache. The deduplication thread, which executes in a concurrent phase, will
86 // later reuse or free the underlying memory for these entries.
87 //
88 // The cache allows for single-threaded allocations and multi-threaded frees.
89 // Allocations are synchronized by StringDedupTable_lock as part of a table
90 // modification.
91 //
92 class G1StringDedupEntryCache : public CHeapObj<mtGC> {
93 private:
94 // One cache/overflow list per GC worker to allow lock less freeing of
95 // entries while doing a parallel scan of the table. Using PaddedEnd to
96 // avoid false sharing.
97 size_t _nlists;
98 size_t _max_list_length;
99 PaddedEnd<G1StringDedupEntryList>* _cached;
100 PaddedEnd<G1StringDedupEntryList>* _overflowed;
102 public:
103 G1StringDedupEntryCache(size_t max_size);
104 ~G1StringDedupEntryCache();
106 // Set max number of table entries to cache.
107 void set_max_size(size_t max_size);
109 // Get a table entry from the cache, or allocate a new entry if the cache is empty.
110 G1StringDedupEntry* alloc();
112 // Insert a table entry into the cache.
113 void free(G1StringDedupEntry* entry, uint worker_id);
115 // Returns current number of entries in the cache.
116 size_t size();
118 // Deletes overflowed entries.
119 void delete_overflowed();
120 };
122 G1StringDedupEntryCache::G1StringDedupEntryCache(size_t max_size) :
123 _nlists(MAX2(ParallelGCThreads, (size_t)1)),
124 _max_list_length(0),
125 _cached(PaddedArray<G1StringDedupEntryList, mtGC>::create_unfreeable((uint)_nlists)),
126 _overflowed(PaddedArray<G1StringDedupEntryList, mtGC>::create_unfreeable((uint)_nlists)) {
127 set_max_size(max_size);
128 }
130 G1StringDedupEntryCache::~G1StringDedupEntryCache() {
131 ShouldNotReachHere();
132 }
134 void G1StringDedupEntryCache::set_max_size(size_t size) {
135 _max_list_length = size / _nlists;
136 }
138 G1StringDedupEntry* G1StringDedupEntryCache::alloc() {
139 for (size_t i = 0; i < _nlists; i++) {
140 G1StringDedupEntry* entry = _cached[i].remove();
141 if (entry != NULL) {
142 return entry;
143 }
144 }
145 return new G1StringDedupEntry();
146 }
148 void G1StringDedupEntryCache::free(G1StringDedupEntry* entry, uint worker_id) {
149 assert(entry->obj() != NULL, "Double free");
150 assert(worker_id < _nlists, "Invalid worker id");
152 entry->set_obj(NULL);
153 entry->set_hash(0);
155 if (_cached[worker_id].length() < _max_list_length) {
156 // Cache is not full
157 _cached[worker_id].add(entry);
158 } else {
159 // Cache is full, add to overflow list for later deletion
160 _overflowed[worker_id].add(entry);
161 }
162 }
164 size_t G1StringDedupEntryCache::size() {
165 size_t size = 0;
166 for (size_t i = 0; i < _nlists; i++) {
167 size += _cached[i].length();
168 }
169 return size;
170 }
172 void G1StringDedupEntryCache::delete_overflowed() {
173 double start = os::elapsedTime();
174 uintx count = 0;
176 for (size_t i = 0; i < _nlists; i++) {
177 G1StringDedupEntry* entry;
179 {
180 // The overflow list can be modified during safepoints, therefore
181 // we temporarily join the suspendible thread set while removing
182 // all entries from the list.
183 SuspendibleThreadSetJoiner sts_join;
184 entry = _overflowed[i].remove_all();
185 }
187 // Delete all entries
188 while (entry != NULL) {
189 G1StringDedupEntry* next = entry->next();
190 delete entry;
191 entry = next;
192 count++;
193 }
194 }
196 double end = os::elapsedTime();
197 if (PrintStringDeduplicationStatistics) {
198 gclog_or_tty->print_cr("[GC concurrent-string-deduplication, deleted " UINTX_FORMAT " entries, " G1_STRDEDUP_TIME_FORMAT "]", count, end - start);
199 }
200 }
202 G1StringDedupTable* G1StringDedupTable::_table = NULL;
203 G1StringDedupEntryCache* G1StringDedupTable::_entry_cache = NULL;
205 const size_t G1StringDedupTable::_min_size = (1 << 10); // 1024
206 const size_t G1StringDedupTable::_max_size = (1 << 24); // 16777216
207 const double G1StringDedupTable::_grow_load_factor = 2.0; // Grow table at 200% load
208 const double G1StringDedupTable::_shrink_load_factor = _grow_load_factor / 3.0; // Shrink table at 67% load
209 const double G1StringDedupTable::_max_cache_factor = 0.1; // Cache a maximum of 10% of the table size
210 const uintx G1StringDedupTable::_rehash_multiple = 60; // Hash bucket has 60 times more collisions than expected
211 const uintx G1StringDedupTable::_rehash_threshold = (uintx)(_rehash_multiple * _grow_load_factor);
213 uintx G1StringDedupTable::_entries_added = 0;
214 uintx G1StringDedupTable::_entries_removed = 0;
215 uintx G1StringDedupTable::_resize_count = 0;
216 uintx G1StringDedupTable::_rehash_count = 0;
218 G1StringDedupTable::G1StringDedupTable(size_t size, jint hash_seed) :
219 _size(size),
220 _entries(0),
221 _grow_threshold((uintx)(size * _grow_load_factor)),
222 _shrink_threshold((uintx)(size * _shrink_load_factor)),
223 _rehash_needed(false),
224 _hash_seed(hash_seed) {
225 assert(is_power_of_2(size), "Table size must be a power of 2");
226 _buckets = NEW_C_HEAP_ARRAY(G1StringDedupEntry*, _size, mtGC);
227 memset(_buckets, 0, _size * sizeof(G1StringDedupEntry*));
228 }
230 G1StringDedupTable::~G1StringDedupTable() {
231 FREE_C_HEAP_ARRAY(G1StringDedupEntry*, _buckets, mtGC);
232 }
234 void G1StringDedupTable::create() {
235 assert(_table == NULL, "One string deduplication table allowed");
236 _entry_cache = new G1StringDedupEntryCache((size_t)(_min_size * _max_cache_factor));
237 _table = new G1StringDedupTable(_min_size);
238 }
240 void G1StringDedupTable::add(typeArrayOop value, unsigned int hash, G1StringDedupEntry** list) {
241 G1StringDedupEntry* entry = _entry_cache->alloc();
242 entry->set_obj(value);
243 entry->set_hash(hash);
244 entry->set_next(*list);
245 *list = entry;
246 _entries++;
247 }
249 void G1StringDedupTable::remove(G1StringDedupEntry** pentry, uint worker_id) {
250 G1StringDedupEntry* entry = *pentry;
251 *pentry = entry->next();
252 _entry_cache->free(entry, worker_id);
253 }
255 void G1StringDedupTable::transfer(G1StringDedupEntry** pentry, G1StringDedupTable* dest) {
256 G1StringDedupEntry* entry = *pentry;
257 *pentry = entry->next();
258 unsigned int hash = entry->hash();
259 size_t index = dest->hash_to_index(hash);
260 G1StringDedupEntry** list = dest->bucket(index);
261 entry->set_next(*list);
262 *list = entry;
263 }
265 bool G1StringDedupTable::equals(typeArrayOop value1, typeArrayOop value2) {
266 return (value1 == value2 ||
267 (value1->length() == value2->length() &&
268 (!memcmp(value1->base(T_CHAR),
269 value2->base(T_CHAR),
270 value1->length() * sizeof(jchar)))));
271 }
273 typeArrayOop G1StringDedupTable::lookup(typeArrayOop value, unsigned int hash,
274 G1StringDedupEntry** list, uintx &count) {
275 for (G1StringDedupEntry* entry = *list; entry != NULL; entry = entry->next()) {
276 if (entry->hash() == hash) {
277 typeArrayOop existing_value = entry->obj();
278 if (equals(value, existing_value)) {
279 // Match found
280 return existing_value;
281 }
282 }
283 count++;
284 }
286 // Not found
287 return NULL;
288 }
290 typeArrayOop G1StringDedupTable::lookup_or_add_inner(typeArrayOop value, unsigned int hash) {
291 size_t index = hash_to_index(hash);
292 G1StringDedupEntry** list = bucket(index);
293 uintx count = 0;
295 // Lookup in list
296 typeArrayOop existing_value = lookup(value, hash, list, count);
298 // Check if rehash is needed
299 if (count > _rehash_threshold) {
300 _rehash_needed = true;
301 }
303 if (existing_value == NULL) {
304 // Not found, add new entry
305 add(value, hash, list);
307 // Update statistics
308 _entries_added++;
309 }
311 return existing_value;
312 }
314 unsigned int G1StringDedupTable::hash_code(typeArrayOop value) {
315 unsigned int hash;
316 int length = value->length();
317 const jchar* data = (jchar*)value->base(T_CHAR);
319 if (use_java_hash()) {
320 hash = java_lang_String::hash_code(data, length);
321 } else {
322 hash = AltHashing::murmur3_32(_table->_hash_seed, data, length);
323 }
325 return hash;
326 }
328 void G1StringDedupTable::deduplicate(oop java_string, G1StringDedupStat& stat) {
329 assert(java_lang_String::is_instance(java_string), "Must be a string");
330 No_Safepoint_Verifier nsv;
332 stat.inc_inspected();
334 typeArrayOop value = java_lang_String::value(java_string);
335 if (value == NULL) {
336 // String has no value
337 stat.inc_skipped();
338 return;
339 }
341 unsigned int hash = 0;
343 if (use_java_hash()) {
344 // Get hash code from cache
345 hash = java_lang_String::hash(java_string);
346 }
348 if (hash == 0) {
349 // Compute hash
350 hash = hash_code(value);
351 stat.inc_hashed();
352 }
354 if (use_java_hash() && hash != 0) {
355 // Store hash code in cache
356 java_lang_String::set_hash(java_string, hash);
357 }
359 typeArrayOop existing_value = lookup_or_add(value, hash);
360 if (existing_value == value) {
361 // Same value, already known
362 stat.inc_known();
363 return;
364 }
366 // Get size of value array
367 uintx size_in_bytes = value->size() * HeapWordSize;
368 stat.inc_new(size_in_bytes);
370 if (existing_value != NULL) {
371 // Enqueue the reference to make sure it is kept alive. Concurrent mark might
372 // otherwise declare it dead if there are no other strong references to this object.
373 G1SATBCardTableModRefBS::enqueue(existing_value);
375 // Existing value found, deduplicate string
376 java_lang_String::set_value(java_string, existing_value);
378 if (G1CollectedHeap::heap()->is_in_young(value)) {
379 stat.inc_deduped_young(size_in_bytes);
380 } else {
381 stat.inc_deduped_old(size_in_bytes);
382 }
383 }
384 }
386 G1StringDedupTable* G1StringDedupTable::prepare_resize() {
387 size_t size = _table->_size;
389 // Check if the hashtable needs to be resized
390 if (_table->_entries > _table->_grow_threshold) {
391 // Grow table, double the size
392 size *= 2;
393 if (size > _max_size) {
394 // Too big, don't resize
395 return NULL;
396 }
397 } else if (_table->_entries < _table->_shrink_threshold) {
398 // Shrink table, half the size
399 size /= 2;
400 if (size < _min_size) {
401 // Too small, don't resize
402 return NULL;
403 }
404 } else if (StringDeduplicationResizeALot) {
405 // Force grow
406 size *= 2;
407 if (size > _max_size) {
408 // Too big, force shrink instead
409 size /= 4;
410 }
411 } else {
412 // Resize not needed
413 return NULL;
414 }
416 // Update statistics
417 _resize_count++;
419 // Update max cache size
420 _entry_cache->set_max_size((size_t)(size * _max_cache_factor));
422 // Allocate the new table. The new table will be populated by workers
423 // calling unlink_or_oops_do() and finally installed by finish_resize().
424 return new G1StringDedupTable(size, _table->_hash_seed);
425 }
427 void G1StringDedupTable::finish_resize(G1StringDedupTable* resized_table) {
428 assert(resized_table != NULL, "Invalid table");
430 resized_table->_entries = _table->_entries;
432 // Free old table
433 delete _table;
435 // Install new table
436 _table = resized_table;
437 }
439 void G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id) {
440 // The table is divided into partitions to allow lock-less parallel processing by
441 // multiple worker threads. A worker thread first claims a partition, which ensures
442 // exclusive access to that part of the table, then continues to process it. To allow
443 // shrinking of the table in parallel we also need to make sure that the same worker
444 // thread processes all partitions where entries will hash to the same destination
445 // partition. Since the table size is always a power of two and we always shrink by
446 // dividing the table in half, we know that for a given partition there is only one
447 // other partition whoes entries will hash to the same destination partition. That
448 // other partition is always the sibling partition in the second half of the table.
449 // For example, if the table is divided into 8 partitions, the sibling of partition 0
450 // is partition 4, the sibling of partition 1 is partition 5, etc.
451 size_t table_half = _table->_size / 2;
453 // Let each partition be one page worth of buckets
454 size_t partition_size = MIN2(table_half, os::vm_page_size() / sizeof(G1StringDedupEntry*));
455 assert(table_half % partition_size == 0, "Invalid partition size");
457 // Number of entries removed during the scan
458 uintx removed = 0;
460 for (;;) {
461 // Grab next partition to scan
462 size_t partition_begin = cl->claim_table_partition(partition_size);
463 size_t partition_end = partition_begin + partition_size;
464 if (partition_begin >= table_half) {
465 // End of table
466 break;
467 }
469 // Scan the partition followed by the sibling partition in the second half of the table
470 removed += unlink_or_oops_do(cl, partition_begin, partition_end, worker_id);
471 removed += unlink_or_oops_do(cl, table_half + partition_begin, table_half + partition_end, worker_id);
472 }
474 // Delayed update to avoid contention on the table lock
475 if (removed > 0) {
476 MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag);
477 _table->_entries -= removed;
478 _entries_removed += removed;
479 }
480 }
482 uintx G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl,
483 size_t partition_begin,
484 size_t partition_end,
485 uint worker_id) {
486 uintx removed = 0;
487 for (size_t bucket = partition_begin; bucket < partition_end; bucket++) {
488 G1StringDedupEntry** entry = _table->bucket(bucket);
489 while (*entry != NULL) {
490 oop* p = (oop*)(*entry)->obj_addr();
491 if (cl->is_alive(*p)) {
492 cl->keep_alive(p);
493 if (cl->is_resizing()) {
494 // We are resizing the table, transfer entry to the new table
495 _table->transfer(entry, cl->resized_table());
496 } else {
497 if (cl->is_rehashing()) {
498 // We are rehashing the table, rehash the entry but keep it
499 // in the table. We can't transfer entries into the new table
500 // at this point since we don't have exclusive access to all
501 // destination partitions. finish_rehash() will do a single
502 // threaded transfer of all entries.
503 typeArrayOop value = (typeArrayOop)*p;
504 unsigned int hash = hash_code(value);
505 (*entry)->set_hash(hash);
506 }
508 // Move to next entry
509 entry = (*entry)->next_addr();
510 }
511 } else {
512 // Not alive, remove entry from table
513 _table->remove(entry, worker_id);
514 removed++;
515 }
516 }
517 }
519 return removed;
520 }
522 G1StringDedupTable* G1StringDedupTable::prepare_rehash() {
523 if (!_table->_rehash_needed && !StringDeduplicationRehashALot) {
524 // Rehash not needed
525 return NULL;
526 }
528 // Update statistics
529 _rehash_count++;
531 // Compute new hash seed
532 _table->_hash_seed = AltHashing::compute_seed();
534 // Allocate the new table, same size and hash seed
535 return new G1StringDedupTable(_table->_size, _table->_hash_seed);
536 }
538 void G1StringDedupTable::finish_rehash(G1StringDedupTable* rehashed_table) {
539 assert(rehashed_table != NULL, "Invalid table");
541 // Move all newly rehashed entries into the correct buckets in the new table
542 for (size_t bucket = 0; bucket < _table->_size; bucket++) {
543 G1StringDedupEntry** entry = _table->bucket(bucket);
544 while (*entry != NULL) {
545 _table->transfer(entry, rehashed_table);
546 }
547 }
549 rehashed_table->_entries = _table->_entries;
551 // Free old table
552 delete _table;
554 // Install new table
555 _table = rehashed_table;
556 }
558 void G1StringDedupTable::verify() {
559 for (size_t bucket = 0; bucket < _table->_size; bucket++) {
560 // Verify entries
561 G1StringDedupEntry** entry = _table->bucket(bucket);
562 while (*entry != NULL) {
563 typeArrayOop value = (*entry)->obj();
564 guarantee(value != NULL, "Object must not be NULL");
565 guarantee(Universe::heap()->is_in_reserved(value), "Object must be on the heap");
566 guarantee(!value->is_forwarded(), "Object must not be forwarded");
567 guarantee(value->is_typeArray(), "Object must be a typeArrayOop");
568 unsigned int hash = hash_code(value);
569 guarantee((*entry)->hash() == hash, "Table entry has inorrect hash");
570 guarantee(_table->hash_to_index(hash) == bucket, "Table entry has incorrect index");
571 entry = (*entry)->next_addr();
572 }
574 // Verify that we do not have entries with identical oops or identical arrays.
575 // We only need to compare entries in the same bucket. If the same oop or an
576 // identical array has been inserted more than once into different/incorrect
577 // buckets the verification step above will catch that.
578 G1StringDedupEntry** entry1 = _table->bucket(bucket);
579 while (*entry1 != NULL) {
580 typeArrayOop value1 = (*entry1)->obj();
581 G1StringDedupEntry** entry2 = (*entry1)->next_addr();
582 while (*entry2 != NULL) {
583 typeArrayOop value2 = (*entry2)->obj();
584 guarantee(!equals(value1, value2), "Table entries must not have identical arrays");
585 entry2 = (*entry2)->next_addr();
586 }
587 entry1 = (*entry1)->next_addr();
588 }
589 }
590 }
592 void G1StringDedupTable::clean_entry_cache() {
593 _entry_cache->delete_overflowed();
594 }
596 void G1StringDedupTable::print_statistics(outputStream* st) {
597 st->print_cr(
598 " [Table]\n"
599 " [Memory Usage: " G1_STRDEDUP_BYTES_FORMAT_NS "]\n"
600 " [Size: " SIZE_FORMAT ", Min: " SIZE_FORMAT ", Max: " SIZE_FORMAT "]\n"
601 " [Entries: " UINTX_FORMAT ", Load: " G1_STRDEDUP_PERCENT_FORMAT_NS ", Cached: " UINTX_FORMAT ", Added: " UINTX_FORMAT ", Removed: " UINTX_FORMAT "]\n"
602 " [Resize Count: " UINTX_FORMAT ", Shrink Threshold: " UINTX_FORMAT "(" G1_STRDEDUP_PERCENT_FORMAT_NS "), Grow Threshold: " UINTX_FORMAT "(" G1_STRDEDUP_PERCENT_FORMAT_NS ")]\n"
603 " [Rehash Count: " UINTX_FORMAT ", Rehash Threshold: " UINTX_FORMAT ", Hash Seed: 0x%x]\n"
604 " [Age Threshold: " UINTX_FORMAT "]",
605 G1_STRDEDUP_BYTES_PARAM(_table->_size * sizeof(G1StringDedupEntry*) + (_table->_entries + _entry_cache->size()) * sizeof(G1StringDedupEntry)),
606 _table->_size, _min_size, _max_size,
607 _table->_entries, (double)_table->_entries / (double)_table->_size * 100.0, _entry_cache->size(), _entries_added, _entries_removed,
608 _resize_count, _table->_shrink_threshold, _shrink_load_factor * 100.0, _table->_grow_threshold, _grow_load_factor * 100.0,
609 _rehash_count, _rehash_threshold, _table->_hash_seed,
610 StringDeduplicationAgeThreshold);
611 }