Mercurial > jdk8-mips64-public > hotspot / changeset
changeset
8048268: G1 Code Root Migration performs poorly
Fri, 29 Aug 2014 13:12:21 +0200
- author
- mgerdin
- date
- Fri, 29 Aug 2014 13:12:21 +0200
- changeset 7208
- 7baf47cb97cb
- parent 7207
- 152cf4afc11f
- child 7209
- 58925d1f325e
8048268: G1 Code Root Migration performs poorly
Summary: Replace G1CodeRootSet with a Hashtable based implementation, merge Code Root Migration phase into Code Root Scanning
Reviewed-by: jmasa, brutisso, tschatzl
1.1 --- a/src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.cpp Fri Aug 29 13:08:01 2014 +0200 1.2 +++ b/src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.cpp Fri Aug 29 13:12:21 2014 +0200 1.3 @@ -22,372 +22,375 @@ 1.4 * 1.5 */ 1.6 1.7 - 1.8 #include "precompiled.hpp" 1.9 +#include "code/codeCache.hpp" 1.10 #include "code/nmethod.hpp" 1.11 #include "gc_implementation/g1/g1CodeCacheRemSet.hpp" 1.12 +#include "gc_implementation/g1/heapRegion.hpp" 1.13 +#include "memory/heap.hpp" 1.14 #include "memory/iterator.hpp" 1.15 +#include "oops/oop.inline.hpp" 1.16 +#include "utilities/hashtable.inline.hpp" 1.17 +#include "utilities/stack.inline.hpp" 1.18 1.19 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC 1.20 1.21 -G1CodeRootChunk::G1CodeRootChunk() : _top(NULL), _next(NULL), _prev(NULL), _free(NULL) { 1.22 - _top = bottom(); 1.23 +class CodeRootSetTable : public Hashtable<nmethod*, mtGC> { 1.24 + friend class G1CodeRootSetTest; 1.25 + typedef HashtableEntry<nmethod*, mtGC> Entry; 1.26 + 1.27 + static CodeRootSetTable* volatile _purge_list; 1.28 + 1.29 + CodeRootSetTable* _purge_next; 1.30 + 1.31 + unsigned int compute_hash(nmethod* nm) { 1.32 + uintptr_t hash = (uintptr_t)nm; 1.33 + return hash ^ (hash >> 7); // code heap blocks are 128byte aligned 1.34 + } 1.35 + 1.36 + Entry* new_entry(nmethod* nm); 1.37 + 1.38 + public: 1.39 + CodeRootSetTable(int size) : Hashtable<nmethod*, mtGC>(size, sizeof(Entry)), _purge_next(NULL) {} 1.40 + ~CodeRootSetTable(); 1.41 + 1.42 + // Needs to be protected locks 1.43 + bool add(nmethod* nm); 1.44 + bool remove(nmethod* nm); 1.45 + 1.46 + // Can be called without locking 1.47 + bool contains(nmethod* nm); 1.48 + 1.49 + int entry_size() const { return BasicHashtable<mtGC>::entry_size(); } 1.50 + 1.51 + void copy_to(CodeRootSetTable* new_table); 1.52 + void nmethods_do(CodeBlobClosure* blk); 1.53 + 1.54 + template<typename CB> 1.55 + void remove_if(CB& should_remove); 1.56 + 1.57 + static void purge_list_append(CodeRootSetTable* tbl); 1.58 + static void purge(); 1.59 + 1.60 + static size_t static_mem_size() { 1.61 + return sizeof(_purge_list); 1.62 + } 1.63 +}; 1.64 + 1.65 +CodeRootSetTable* volatile CodeRootSetTable::_purge_list = NULL; 1.66 + 1.67 +CodeRootSetTable::Entry* CodeRootSetTable::new_entry(nmethod* nm) { 1.68 + unsigned int hash = compute_hash(nm); 1.69 + Entry* entry = (Entry*) new_entry_free_list(); 1.70 + if (entry == NULL) { 1.71 + entry = (Entry*) NEW_C_HEAP_ARRAY2(char, entry_size(), mtGC, CURRENT_PC); 1.72 + } 1.73 + entry->set_next(NULL); 1.74 + entry->set_hash(hash); 1.75 + entry->set_literal(nm); 1.76 + return entry; 1.77 } 1.78 1.79 -void G1CodeRootChunk::reset() { 1.80 - _next = _prev = NULL; 1.81 - _free = NULL; 1.82 - _top = bottom(); 1.83 -} 1.84 - 1.85 -void G1CodeRootChunk::nmethods_do(CodeBlobClosure* cl) { 1.86 - NmethodOrLink* cur = bottom(); 1.87 - while (cur != _top) { 1.88 - if (is_nmethod(cur)) { 1.89 - cl->do_code_blob(cur->_nmethod); 1.90 +CodeRootSetTable::~CodeRootSetTable() { 1.91 + for (int index = 0; index < table_size(); ++index) { 1.92 + for (Entry* e = bucket(index); e != NULL; ) { 1.93 + Entry* to_remove = e; 1.94 + // read next before freeing. 1.95 + e = e->next(); 1.96 + unlink_entry(to_remove); 1.97 + FREE_C_HEAP_ARRAY(char, to_remove, mtGC); 1.98 } 1.99 - cur++; 1.100 + } 1.101 + assert(number_of_entries() == 0, "should have removed all entries"); 1.102 + free_buckets(); 1.103 + for (BasicHashtableEntry<mtGC>* e = new_entry_free_list(); e != NULL; e = new_entry_free_list()) { 1.104 + FREE_C_HEAP_ARRAY(char, e, mtGC); 1.105 } 1.106 } 1.107 1.108 -bool G1CodeRootChunk::remove_lock_free(nmethod* method) { 1.109 - NmethodOrLink* cur = bottom(); 1.110 +bool CodeRootSetTable::add(nmethod* nm) { 1.111 + if (!contains(nm)) { 1.112 + Entry* e = new_entry(nm); 1.113 + int index = hash_to_index(e->hash()); 1.114 + add_entry(index, e); 1.115 + return true; 1.116 + } 1.117 + return false; 1.118 +} 1.119 1.120 - for (NmethodOrLink* cur = bottom(); cur != _top; cur++) { 1.121 - if (cur->_nmethod == method) { 1.122 - bool result = Atomic::cmpxchg_ptr(NULL, &cur->_nmethod, method) == method; 1.123 - 1.124 - if (!result) { 1.125 - // Someone else cleared out this entry. 1.126 - return false; 1.127 - } 1.128 - 1.129 - // The method was cleared. Time to link it into the free list. 1.130 - NmethodOrLink* prev_free; 1.131 - do { 1.132 - prev_free = (NmethodOrLink*)_free; 1.133 - cur->_link = prev_free; 1.134 - } while (Atomic::cmpxchg_ptr(cur, &_free, prev_free) != prev_free); 1.135 - 1.136 +bool CodeRootSetTable::contains(nmethod* nm) { 1.137 + int index = hash_to_index(compute_hash(nm)); 1.138 + for (Entry* e = bucket(index); e != NULL; e = e->next()) { 1.139 + if (e->literal() == nm) { 1.140 return true; 1.141 } 1.142 } 1.143 - 1.144 return false; 1.145 } 1.146 1.147 -G1CodeRootChunkManager::G1CodeRootChunkManager() : _free_list(), _num_chunks_handed_out(0) { 1.148 - _free_list.initialize(); 1.149 - _free_list.set_size(G1CodeRootChunk::word_size()); 1.150 +bool CodeRootSetTable::remove(nmethod* nm) { 1.151 + int index = hash_to_index(compute_hash(nm)); 1.152 + Entry* previous = NULL; 1.153 + for (Entry* e = bucket(index); e != NULL; previous = e, e = e->next()) { 1.154 + if (e->literal() == nm) { 1.155 + if (previous != NULL) { 1.156 + previous->set_next(e->next()); 1.157 + } else { 1.158 + set_entry(index, e->next()); 1.159 + } 1.160 + free_entry(e); 1.161 + return true; 1.162 + } 1.163 + } 1.164 + return false; 1.165 } 1.166 1.167 -size_t G1CodeRootChunkManager::fl_mem_size() { 1.168 - return _free_list.count() * _free_list.size(); 1.169 +void CodeRootSetTable::copy_to(CodeRootSetTable* new_table) { 1.170 + for (int index = 0; index < table_size(); ++index) { 1.171 + for (Entry* e = bucket(index); e != NULL; e = e->next()) { 1.172 + new_table->add(e->literal()); 1.173 + } 1.174 + } 1.175 + new_table->copy_freelist(this); 1.176 } 1.177 1.178 -void G1CodeRootChunkManager::free_all_chunks(FreeList<G1CodeRootChunk>* list) { 1.179 - _num_chunks_handed_out -= list->count(); 1.180 - _free_list.prepend(list); 1.181 -} 1.182 - 1.183 -void G1CodeRootChunkManager::free_chunk(G1CodeRootChunk* chunk) { 1.184 - _free_list.return_chunk_at_head(chunk); 1.185 - _num_chunks_handed_out--; 1.186 -} 1.187 - 1.188 -void G1CodeRootChunkManager::purge_chunks(size_t keep_ratio) { 1.189 - size_t keep = _num_chunks_handed_out * keep_ratio / 100; 1.190 - if (keep >= (size_t)_free_list.count()) { 1.191 - return; 1.192 - } 1.193 - 1.194 - FreeList<G1CodeRootChunk> temp; 1.195 - temp.initialize(); 1.196 - temp.set_size(G1CodeRootChunk::word_size()); 1.197 - 1.198 - _free_list.getFirstNChunksFromList((size_t)_free_list.count() - keep, &temp); 1.199 - 1.200 - G1CodeRootChunk* cur = temp.get_chunk_at_head(); 1.201 - while (cur != NULL) { 1.202 - delete cur; 1.203 - cur = temp.get_chunk_at_head(); 1.204 +void CodeRootSetTable::nmethods_do(CodeBlobClosure* blk) { 1.205 + for (int index = 0; index < table_size(); ++index) { 1.206 + for (Entry* e = bucket(index); e != NULL; e = e->next()) { 1.207 + blk->do_code_blob(e->literal()); 1.208 + } 1.209 } 1.210 } 1.211 1.212 -size_t G1CodeRootChunkManager::static_mem_size() { 1.213 - return sizeof(G1CodeRootChunkManager); 1.214 +template<typename CB> 1.215 +void CodeRootSetTable::remove_if(CB& should_remove) { 1.216 + for (int index = 0; index < table_size(); ++index) { 1.217 + Entry* previous = NULL; 1.218 + Entry* e = bucket(index); 1.219 + while (e != NULL) { 1.220 + Entry* next = e->next(); 1.221 + if (should_remove(e->literal())) { 1.222 + if (previous != NULL) { 1.223 + previous->set_next(next); 1.224 + } else { 1.225 + set_entry(index, next); 1.226 + } 1.227 + free_entry(e); 1.228 + } else { 1.229 + previous = e; 1.230 + } 1.231 + e = next; 1.232 + } 1.233 + } 1.234 } 1.235 1.236 +G1CodeRootSet::~G1CodeRootSet() { 1.237 + delete _table; 1.238 +} 1.239 1.240 -G1CodeRootChunk* G1CodeRootChunkManager::new_chunk() { 1.241 - G1CodeRootChunk* result = _free_list.get_chunk_at_head(); 1.242 - if (result == NULL) { 1.243 - result = new G1CodeRootChunk(); 1.244 +CodeRootSetTable* G1CodeRootSet::load_acquire_table() { 1.245 + return (CodeRootSetTable*) OrderAccess::load_ptr_acquire(&_table); 1.246 +} 1.247 + 1.248 +void G1CodeRootSet::allocate_small_table() { 1.249 + _table = new CodeRootSetTable(SmallSize); 1.250 +} 1.251 + 1.252 +void CodeRootSetTable::purge_list_append(CodeRootSetTable* table) { 1.253 + for (;;) { 1.254 + table->_purge_next = _purge_list; 1.255 + CodeRootSetTable* old = (CodeRootSetTable*) Atomic::cmpxchg_ptr(table, &_purge_list, table->_purge_next); 1.256 + if (old == table->_purge_next) { 1.257 + break; 1.258 + } 1.259 } 1.260 - _num_chunks_handed_out++; 1.261 - result->reset(); 1.262 - return result; 1.263 +} 1.264 + 1.265 +void CodeRootSetTable::purge() { 1.266 + CodeRootSetTable* table = _purge_list; 1.267 + _purge_list = NULL; 1.268 + while (table != NULL) { 1.269 + CodeRootSetTable* to_purge = table; 1.270 + table = table->_purge_next; 1.271 + delete to_purge; 1.272 + } 1.273 +} 1.274 + 1.275 +void G1CodeRootSet::move_to_large() { 1.276 + CodeRootSetTable* temp = new CodeRootSetTable(LargeSize); 1.277 + 1.278 + _table->copy_to(temp); 1.279 + 1.280 + CodeRootSetTable::purge_list_append(_table); 1.281 + 1.282 + OrderAccess::release_store_ptr(&_table, temp); 1.283 +} 1.284 + 1.285 + 1.286 +void G1CodeRootSet::purge() { 1.287 + CodeRootSetTable::purge(); 1.288 +} 1.289 + 1.290 +size_t G1CodeRootSet::static_mem_size() { 1.291 + return CodeRootSetTable::static_mem_size(); 1.292 +} 1.293 + 1.294 +void G1CodeRootSet::add(nmethod* method) { 1.295 + bool added = false; 1.296 + if (is_empty()) { 1.297 + allocate_small_table(); 1.298 + } 1.299 + added = _table->add(method); 1.300 + if (_length == Threshold) { 1.301 + move_to_large(); 1.302 + } 1.303 + if (added) { 1.304 + ++_length; 1.305 + } 1.306 +} 1.307 + 1.308 +bool G1CodeRootSet::remove(nmethod* method) { 1.309 + bool removed = false; 1.310 + if (_table != NULL) { 1.311 + removed = _table->remove(method); 1.312 + } 1.313 + if (removed) { 1.314 + _length--; 1.315 + if (_length == 0) { 1.316 + clear(); 1.317 + } 1.318 + } 1.319 + return removed; 1.320 +} 1.321 + 1.322 +bool G1CodeRootSet::contains(nmethod* method) { 1.323 + CodeRootSetTable* table = load_acquire_table(); 1.324 + if (table != NULL) { 1.325 + return table->contains(method); 1.326 + } 1.327 + return false; 1.328 +} 1.329 + 1.330 +void G1CodeRootSet::clear() { 1.331 + delete _table; 1.332 + _table = NULL; 1.333 + _length = 0; 1.334 +} 1.335 + 1.336 +size_t G1CodeRootSet::mem_size() { 1.337 + return sizeof(*this) + 1.338 + (_table != NULL ? sizeof(CodeRootSetTable) + _table->entry_size() * _length : 0); 1.339 +} 1.340 + 1.341 +void G1CodeRootSet::nmethods_do(CodeBlobClosure* blk) const { 1.342 + if (_table != NULL) { 1.343 + _table->nmethods_do(blk); 1.344 + } 1.345 +} 1.346 + 1.347 +class CleanCallback : public StackObj { 1.348 + class PointsIntoHRDetectionClosure : public OopClosure { 1.349 + HeapRegion* _hr; 1.350 + public: 1.351 + bool _points_into; 1.352 + PointsIntoHRDetectionClosure(HeapRegion* hr) : _hr(hr), _points_into(false) {} 1.353 + 1.354 + void do_oop(narrowOop* o) { 1.355 + do_oop_work(o); 1.356 + } 1.357 + 1.358 + void do_oop(oop* o) { 1.359 + do_oop_work(o); 1.360 + } 1.361 + 1.362 + template <typename T> 1.363 + void do_oop_work(T* p) { 1.364 + if (_hr->is_in(oopDesc::load_decode_heap_oop(p))) { 1.365 + _points_into = true; 1.366 + } 1.367 + } 1.368 + }; 1.369 + 1.370 + PointsIntoHRDetectionClosure _detector; 1.371 + CodeBlobToOopClosure _blobs; 1.372 + 1.373 + public: 1.374 + CleanCallback(HeapRegion* hr) : _detector(hr), _blobs(&_detector, !CodeBlobToOopClosure::FixRelocations) {} 1.375 + 1.376 + bool operator() (nmethod* nm) { 1.377 + _detector._points_into = false; 1.378 + _blobs.do_code_blob(nm); 1.379 + return _detector._points_into; 1.380 + } 1.381 +}; 1.382 + 1.383 +void G1CodeRootSet::clean(HeapRegion* owner) { 1.384 + CleanCallback should_clean(owner); 1.385 + if (_table != NULL) { 1.386 + _table->remove_if(should_clean); 1.387 + } 1.388 } 1.389 1.390 #ifndef PRODUCT 1.391 1.392 -size_t G1CodeRootChunkManager::num_chunks_handed_out() const { 1.393 - return _num_chunks_handed_out; 1.394 -} 1.395 +class G1CodeRootSetTest { 1.396 + public: 1.397 + static void test() { 1.398 + { 1.399 + G1CodeRootSet set1; 1.400 + assert(set1.is_empty(), "Code root set must be initially empty but is not."); 1.401 1.402 -size_t G1CodeRootChunkManager::num_free_chunks() const { 1.403 - return (size_t)_free_list.count(); 1.404 + assert(G1CodeRootSet::static_mem_size() == sizeof(void*), 1.405 + err_msg("The code root set's static memory usage is incorrect, "SIZE_FORMAT" bytes", G1CodeRootSet::static_mem_size())); 1.406 + 1.407 + set1.add((nmethod*)1); 1.408 + assert(set1.length() == 1, err_msg("Added exactly one element, but set contains " 1.409 + SIZE_FORMAT" elements", set1.length())); 1.410 + 1.411 + const size_t num_to_add = (size_t)G1CodeRootSet::Threshold + 1; 1.412 + 1.413 + for (size_t i = 1; i <= num_to_add; i++) { 1.414 + set1.add((nmethod*)1); 1.415 + } 1.416 + assert(set1.length() == 1, 1.417 + err_msg("Duplicate detection should not have increased the set size but " 1.418 + "is "SIZE_FORMAT, set1.length())); 1.419 + 1.420 + for (size_t i = 2; i <= num_to_add; i++) { 1.421 + set1.add((nmethod*)(uintptr_t)(i)); 1.422 + } 1.423 + assert(set1.length() == num_to_add, 1.424 + err_msg("After adding in total "SIZE_FORMAT" distinct code roots, they " 1.425 + "need to be in the set, but there are only "SIZE_FORMAT, 1.426 + num_to_add, set1.length())); 1.427 + 1.428 + assert(CodeRootSetTable::_purge_list != NULL, "should have grown to large hashtable"); 1.429 + 1.430 + size_t num_popped = 0; 1.431 + for (size_t i = 1; i <= num_to_add; i++) { 1.432 + bool removed = set1.remove((nmethod*)i); 1.433 + if (removed) { 1.434 + num_popped += 1; 1.435 + } else { 1.436 + break; 1.437 + } 1.438 + } 1.439 + assert(num_popped == num_to_add, 1.440 + err_msg("Managed to pop "SIZE_FORMAT" code roots, but only "SIZE_FORMAT" " 1.441 + "were added", num_popped, num_to_add)); 1.442 + assert(CodeRootSetTable::_purge_list != NULL, "should have grown to large hashtable"); 1.443 + 1.444 + G1CodeRootSet::purge(); 1.445 + 1.446 + assert(CodeRootSetTable::_purge_list == NULL, "should have purged old small tables"); 1.447 + 1.448 + } 1.449 + 1.450 + } 1.451 +}; 1.452 + 1.453 +void TestCodeCacheRemSet_test() { 1.454 + G1CodeRootSetTest::test(); 1.455 } 1.456 1.457 #endif 1.458 - 1.459 -G1CodeRootChunkManager G1CodeRootSet::_default_chunk_manager; 1.460 - 1.461 -void G1CodeRootSet::purge_chunks(size_t keep_ratio) { 1.462 - _default_chunk_manager.purge_chunks(keep_ratio); 1.463 -} 1.464 - 1.465 -size_t G1CodeRootSet::free_chunks_static_mem_size() { 1.466 - return _default_chunk_manager.static_mem_size(); 1.467 -} 1.468 - 1.469 -size_t G1CodeRootSet::free_chunks_mem_size() { 1.470 - return _default_chunk_manager.fl_mem_size(); 1.471 -} 1.472 - 1.473 -G1CodeRootSet::G1CodeRootSet(G1CodeRootChunkManager* manager) : _manager(manager), _list(), _length(0) { 1.474 - if (_manager == NULL) { 1.475 - _manager = &_default_chunk_manager; 1.476 - } 1.477 - _list.initialize(); 1.478 - _list.set_size(G1CodeRootChunk::word_size()); 1.479 -} 1.480 - 1.481 -G1CodeRootSet::~G1CodeRootSet() { 1.482 - clear(); 1.483 -} 1.484 - 1.485 -void G1CodeRootSet::add(nmethod* method) { 1.486 - if (!contains(method)) { 1.487 - // Find the first chunk that isn't full. 1.488 - G1CodeRootChunk* cur = _list.head(); 1.489 - while (cur != NULL) { 1.490 - if (!cur->is_full()) { 1.491 - break; 1.492 - } 1.493 - cur = cur->next(); 1.494 - } 1.495 - 1.496 - // All chunks are full, get a new chunk. 1.497 - if (cur == NULL) { 1.498 - cur = new_chunk(); 1.499 - _list.return_chunk_at_head(cur); 1.500 - } 1.501 - 1.502 - // Add the nmethod. 1.503 - bool result = cur->add(method); 1.504 - 1.505 - guarantee(result, err_msg("Not able to add nmethod "PTR_FORMAT" to newly allocated chunk.", method)); 1.506 - 1.507 - _length++; 1.508 - } 1.509 -} 1.510 - 1.511 -void G1CodeRootSet::remove_lock_free(nmethod* method) { 1.512 - G1CodeRootChunk* found = find(method); 1.513 - if (found != NULL) { 1.514 - bool result = found->remove_lock_free(method); 1.515 - if (result) { 1.516 - Atomic::dec_ptr((volatile intptr_t*)&_length); 1.517 - } 1.518 - } 1.519 - assert(!contains(method), err_msg(PTR_FORMAT" still contains nmethod "PTR_FORMAT, this, method)); 1.520 -} 1.521 - 1.522 -nmethod* G1CodeRootSet::pop() { 1.523 - while (true) { 1.524 - G1CodeRootChunk* cur = _list.head(); 1.525 - if (cur == NULL) { 1.526 - assert(_length == 0, "when there are no chunks, there should be no elements"); 1.527 - return NULL; 1.528 - } 1.529 - nmethod* result = cur->pop(); 1.530 - if (result != NULL) { 1.531 - _length--; 1.532 - return result; 1.533 - } else { 1.534 - free(_list.get_chunk_at_head()); 1.535 - } 1.536 - } 1.537 -} 1.538 - 1.539 -G1CodeRootChunk* G1CodeRootSet::find(nmethod* method) { 1.540 - G1CodeRootChunk* cur = _list.head(); 1.541 - while (cur != NULL) { 1.542 - if (cur->contains(method)) { 1.543 - return cur; 1.544 - } 1.545 - cur = (G1CodeRootChunk*)cur->next(); 1.546 - } 1.547 - return NULL; 1.548 -} 1.549 - 1.550 -void G1CodeRootSet::free(G1CodeRootChunk* chunk) { 1.551 - free_chunk(chunk); 1.552 -} 1.553 - 1.554 -bool G1CodeRootSet::contains(nmethod* method) { 1.555 - return find(method) != NULL; 1.556 -} 1.557 - 1.558 -void G1CodeRootSet::clear() { 1.559 - free_all_chunks(&_list); 1.560 - _length = 0; 1.561 -} 1.562 - 1.563 -void G1CodeRootSet::nmethods_do(CodeBlobClosure* blk) const { 1.564 - G1CodeRootChunk* cur = _list.head(); 1.565 - while (cur != NULL) { 1.566 - cur->nmethods_do(blk); 1.567 - cur = (G1CodeRootChunk*)cur->next(); 1.568 - } 1.569 -} 1.570 - 1.571 -size_t G1CodeRootSet::static_mem_size() { 1.572 - return sizeof(G1CodeRootSet); 1.573 -} 1.574 - 1.575 -size_t G1CodeRootSet::mem_size() { 1.576 - return G1CodeRootSet::static_mem_size() + _list.count() * _list.size(); 1.577 -} 1.578 - 1.579 -#ifndef PRODUCT 1.580 - 1.581 -void G1CodeRootSet::test() { 1.582 - G1CodeRootChunkManager mgr; 1.583 - 1.584 - assert(mgr.num_chunks_handed_out() == 0, "Must not have handed out chunks yet"); 1.585 - 1.586 - assert(G1CodeRootChunkManager::static_mem_size() > sizeof(void*), 1.587 - err_msg("The chunk manager's static memory usage seems too small, is only "SIZE_FORMAT" bytes.", G1CodeRootChunkManager::static_mem_size())); 1.588 - 1.589 - // The number of chunks that we allocate for purge testing. 1.590 - size_t const num_chunks = 10; 1.591 - 1.592 - { 1.593 - G1CodeRootSet set1(&mgr); 1.594 - assert(set1.is_empty(), "Code root set must be initially empty but is not."); 1.595 - 1.596 - assert(G1CodeRootSet::static_mem_size() > sizeof(void*), 1.597 - err_msg("The code root set's static memory usage seems too small, is only "SIZE_FORMAT" bytes", G1CodeRootSet::static_mem_size())); 1.598 - 1.599 - set1.add((nmethod*)1); 1.600 - assert(mgr.num_chunks_handed_out() == 1, 1.601 - err_msg("Must have allocated and handed out one chunk, but handed out " 1.602 - SIZE_FORMAT" chunks", mgr.num_chunks_handed_out())); 1.603 - assert(set1.length() == 1, err_msg("Added exactly one element, but set contains " 1.604 - SIZE_FORMAT" elements", set1.length())); 1.605 - 1.606 - // G1CodeRootChunk::word_size() is larger than G1CodeRootChunk::num_entries which 1.607 - // we cannot access. 1.608 - for (uint i = 0; i < G1CodeRootChunk::word_size() + 1; i++) { 1.609 - set1.add((nmethod*)1); 1.610 - } 1.611 - assert(mgr.num_chunks_handed_out() == 1, 1.612 - err_msg("Duplicate detection must have prevented allocation of further " 1.613 - "chunks but allocated "SIZE_FORMAT, mgr.num_chunks_handed_out())); 1.614 - assert(set1.length() == 1, 1.615 - err_msg("Duplicate detection should not have increased the set size but " 1.616 - "is "SIZE_FORMAT, set1.length())); 1.617 - 1.618 - size_t num_total_after_add = G1CodeRootChunk::word_size() + 1; 1.619 - for (size_t i = 0; i < num_total_after_add - 1; i++) { 1.620 - set1.add((nmethod*)(uintptr_t)(2 + i)); 1.621 - } 1.622 - assert(mgr.num_chunks_handed_out() > 1, 1.623 - "After adding more code roots, more than one additional chunk should have been handed out"); 1.624 - assert(set1.length() == num_total_after_add, 1.625 - err_msg("After adding in total "SIZE_FORMAT" distinct code roots, they " 1.626 - "need to be in the set, but there are only "SIZE_FORMAT, 1.627 - num_total_after_add, set1.length())); 1.628 - 1.629 - size_t num_popped = 0; 1.630 - while (set1.pop() != NULL) { 1.631 - num_popped++; 1.632 - } 1.633 - assert(num_popped == num_total_after_add, 1.634 - err_msg("Managed to pop "SIZE_FORMAT" code roots, but only "SIZE_FORMAT" " 1.635 - "were added", num_popped, num_total_after_add)); 1.636 - assert(mgr.num_chunks_handed_out() == 0, 1.637 - err_msg("After popping all elements, all chunks must have been returned " 1.638 - "but there are still "SIZE_FORMAT" additional", mgr.num_chunks_handed_out())); 1.639 - 1.640 - mgr.purge_chunks(0); 1.641 - assert(mgr.num_free_chunks() == 0, 1.642 - err_msg("After purging everything, the free list must be empty but still " 1.643 - "contains "SIZE_FORMAT" chunks", mgr.num_free_chunks())); 1.644 - 1.645 - // Add some more handed out chunks. 1.646 - size_t i = 0; 1.647 - while (mgr.num_chunks_handed_out() < num_chunks) { 1.648 - set1.add((nmethod*)i); 1.649 - i++; 1.650 - } 1.651 - 1.652 - { 1.653 - // Generate chunks on the free list. 1.654 - G1CodeRootSet set2(&mgr); 1.655 - size_t i = 0; 1.656 - while (mgr.num_chunks_handed_out() < (num_chunks * 2)) { 1.657 - set2.add((nmethod*)i); 1.658 - i++; 1.659 - } 1.660 - // Exit of the scope of the set2 object will call the destructor that generates 1.661 - // num_chunks elements on the free list. 1.662 - } 1.663 - 1.664 - assert(mgr.num_chunks_handed_out() == num_chunks, 1.665 - err_msg("Deletion of the second set must have resulted in giving back " 1.666 - "those, but there are still "SIZE_FORMAT" additional handed out, expecting " 1.667 - SIZE_FORMAT, mgr.num_chunks_handed_out(), num_chunks)); 1.668 - assert(mgr.num_free_chunks() == num_chunks, 1.669 - err_msg("After freeing "SIZE_FORMAT" chunks, they must be on the free list " 1.670 - "but there are only "SIZE_FORMAT, num_chunks, mgr.num_free_chunks())); 1.671 - 1.672 - size_t const test_percentage = 50; 1.673 - mgr.purge_chunks(test_percentage); 1.674 - assert(mgr.num_chunks_handed_out() == num_chunks, 1.675 - err_msg("Purging must not hand out chunks but there are "SIZE_FORMAT, 1.676 - mgr.num_chunks_handed_out())); 1.677 - assert(mgr.num_free_chunks() == (size_t)(mgr.num_chunks_handed_out() * test_percentage / 100), 1.678 - err_msg("Must have purged "SIZE_FORMAT" percent of "SIZE_FORMAT" chunks" 1.679 - "but there are "SIZE_FORMAT, test_percentage, num_chunks, 1.680 - mgr.num_free_chunks())); 1.681 - // Purge the remainder of the chunks on the free list. 1.682 - mgr.purge_chunks(0); 1.683 - assert(mgr.num_free_chunks() == 0, "Free List must be empty"); 1.684 - assert(mgr.num_chunks_handed_out() == num_chunks, 1.685 - err_msg("Expected to be "SIZE_FORMAT" chunks handed out from the first set " 1.686 - "but there are "SIZE_FORMAT, num_chunks, mgr.num_chunks_handed_out())); 1.687 - 1.688 - // Exit of the scope of the set1 object will call the destructor that generates 1.689 - // num_chunks additional elements on the free list. 1.690 - } 1.691 - 1.692 - assert(mgr.num_chunks_handed_out() == 0, 1.693 - err_msg("Deletion of the only set must have resulted in no chunks handed " 1.694 - "out, but there is still "SIZE_FORMAT" handed out", mgr.num_chunks_handed_out())); 1.695 - assert(mgr.num_free_chunks() == num_chunks, 1.696 - err_msg("After freeing "SIZE_FORMAT" chunks, they must be on the free list " 1.697 - "but there are only "SIZE_FORMAT, num_chunks, mgr.num_free_chunks())); 1.698 - 1.699 - // Restore initial state. 1.700 - mgr.purge_chunks(0); 1.701 - assert(mgr.num_free_chunks() == 0, "Free List must be empty"); 1.702 - assert(mgr.num_chunks_handed_out() == 0, "No additional elements must have been handed out yet"); 1.703 -} 1.704 - 1.705 -void TestCodeCacheRemSet_test() { 1.706 - G1CodeRootSet::test(); 1.707 -} 1.708 -#endif
2.1 --- a/src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.hpp Fri Aug 29 13:08:01 2014 +0200 2.2 +++ b/src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.hpp Fri Aug 29 13:12:21 2014 +0200 2.3 @@ -26,222 +26,64 @@ 2.4 #define SHARE_VM_GC_IMPLEMENTATION_G1_G1CODECACHEREMSET_HPP 2.5 2.6 #include "memory/allocation.hpp" 2.7 -#include "memory/freeList.hpp" 2.8 -#include "runtime/globals.hpp" 2.9 2.10 class CodeBlobClosure; 2.11 - 2.12 -// The elements of the G1CodeRootChunk is either: 2.13 -// 1) nmethod pointers 2.14 -// 2) nodes in an internally chained free list 2.15 -typedef union { 2.16 - nmethod* _nmethod; 2.17 - void* _link; 2.18 -} NmethodOrLink; 2.19 - 2.20 -class G1CodeRootChunk : public CHeapObj<mtGC> { 2.21 - private: 2.22 - static const int NUM_ENTRIES = 32; 2.23 - public: 2.24 - G1CodeRootChunk* _next; 2.25 - G1CodeRootChunk* _prev; 2.26 - 2.27 - NmethodOrLink* _top; 2.28 - // First free position within the chunk. 2.29 - volatile NmethodOrLink* _free; 2.30 - 2.31 - NmethodOrLink _data[NUM_ENTRIES]; 2.32 - 2.33 - NmethodOrLink* bottom() const { 2.34 - return (NmethodOrLink*) &(_data[0]); 2.35 - } 2.36 - 2.37 - NmethodOrLink* end() const { 2.38 - return (NmethodOrLink*) &(_data[NUM_ENTRIES]); 2.39 - } 2.40 - 2.41 - bool is_link(NmethodOrLink* nmethod_or_link) { 2.42 - return nmethod_or_link->_link == NULL || 2.43 - (bottom() <= nmethod_or_link->_link 2.44 - && nmethod_or_link->_link < end()); 2.45 - } 2.46 - 2.47 - bool is_nmethod(NmethodOrLink* nmethod_or_link) { 2.48 - return !is_link(nmethod_or_link); 2.49 - } 2.50 - 2.51 - public: 2.52 - G1CodeRootChunk(); 2.53 - ~G1CodeRootChunk() {} 2.54 - 2.55 - static size_t word_size() { return (size_t)(align_size_up_(sizeof(G1CodeRootChunk), HeapWordSize) / HeapWordSize); } 2.56 - 2.57 - // FreeList "interface" methods 2.58 - 2.59 - G1CodeRootChunk* next() const { return _next; } 2.60 - G1CodeRootChunk* prev() const { return _prev; } 2.61 - void set_next(G1CodeRootChunk* v) { _next = v; assert(v != this, "Boom");} 2.62 - void set_prev(G1CodeRootChunk* v) { _prev = v; assert(v != this, "Boom");} 2.63 - void clear_next() { set_next(NULL); } 2.64 - void clear_prev() { set_prev(NULL); } 2.65 - 2.66 - size_t size() const { return word_size(); } 2.67 - 2.68 - void link_next(G1CodeRootChunk* ptr) { set_next(ptr); } 2.69 - void link_prev(G1CodeRootChunk* ptr) { set_prev(ptr); } 2.70 - void link_after(G1CodeRootChunk* ptr) { 2.71 - link_next(ptr); 2.72 - if (ptr != NULL) ptr->link_prev((G1CodeRootChunk*)this); 2.73 - } 2.74 - 2.75 - bool is_free() { return true; } 2.76 - 2.77 - // New G1CodeRootChunk routines 2.78 - 2.79 - void reset(); 2.80 - 2.81 - bool is_empty() const { 2.82 - return _top == bottom(); 2.83 - } 2.84 - 2.85 - bool is_full() const { 2.86 - return _top == end() && _free == NULL; 2.87 - } 2.88 - 2.89 - bool contains(nmethod* method) { 2.90 - NmethodOrLink* cur = bottom(); 2.91 - while (cur != _top) { 2.92 - if (cur->_nmethod == method) return true; 2.93 - cur++; 2.94 - } 2.95 - return false; 2.96 - } 2.97 - 2.98 - bool add(nmethod* method) { 2.99 - if (is_full()) { 2.100 - return false; 2.101 - } 2.102 - 2.103 - if (_free != NULL) { 2.104 - // Take from internally chained free list 2.105 - NmethodOrLink* first_free = (NmethodOrLink*)_free; 2.106 - _free = (NmethodOrLink*)_free->_link; 2.107 - first_free->_nmethod = method; 2.108 - } else { 2.109 - // Take from top. 2.110 - _top->_nmethod = method; 2.111 - _top++; 2.112 - } 2.113 - 2.114 - return true; 2.115 - } 2.116 - 2.117 - bool remove_lock_free(nmethod* method); 2.118 - 2.119 - void nmethods_do(CodeBlobClosure* blk); 2.120 - 2.121 - nmethod* pop() { 2.122 - if (_free != NULL) { 2.123 - // Kill the free list. 2.124 - _free = NULL; 2.125 - } 2.126 - 2.127 - while (!is_empty()) { 2.128 - _top--; 2.129 - if (is_nmethod(_top)) { 2.130 - return _top->_nmethod; 2.131 - } 2.132 - } 2.133 - 2.134 - return NULL; 2.135 - } 2.136 -}; 2.137 - 2.138 -// Manages free chunks. 2.139 -class G1CodeRootChunkManager VALUE_OBJ_CLASS_SPEC { 2.140 - private: 2.141 - // Global free chunk list management 2.142 - FreeList<G1CodeRootChunk> _free_list; 2.143 - // Total number of chunks handed out 2.144 - size_t _num_chunks_handed_out; 2.145 - 2.146 - public: 2.147 - G1CodeRootChunkManager(); 2.148 - 2.149 - G1CodeRootChunk* new_chunk(); 2.150 - void free_chunk(G1CodeRootChunk* chunk); 2.151 - // Free all elements of the given list. 2.152 - void free_all_chunks(FreeList<G1CodeRootChunk>* list); 2.153 - 2.154 - void initialize(); 2.155 - void purge_chunks(size_t keep_ratio); 2.156 - 2.157 - static size_t static_mem_size(); 2.158 - size_t fl_mem_size(); 2.159 - 2.160 -#ifndef PRODUCT 2.161 - size_t num_chunks_handed_out() const; 2.162 - size_t num_free_chunks() const; 2.163 -#endif 2.164 -}; 2.165 +class CodeRootSetTable; 2.166 +class HeapRegion; 2.167 +class nmethod; 2.168 2.169 // Implements storage for a set of code roots. 2.170 // All methods that modify the set are not thread-safe except if otherwise noted. 2.171 class G1CodeRootSet VALUE_OBJ_CLASS_SPEC { 2.172 + friend class G1CodeRootSetTest; 2.173 private: 2.174 - // Global default free chunk manager instance. 2.175 - static G1CodeRootChunkManager _default_chunk_manager; 2.176 2.177 - G1CodeRootChunk* new_chunk() { return _manager->new_chunk(); } 2.178 - void free_chunk(G1CodeRootChunk* chunk) { _manager->free_chunk(chunk); } 2.179 - // Free all elements of the given list. 2.180 - void free_all_chunks(FreeList<G1CodeRootChunk>* list) { _manager->free_all_chunks(list); } 2.181 + const static size_t SmallSize = 32; 2.182 + const static size_t Threshold = 24; 2.183 + const static size_t LargeSize = 512; 2.184 2.185 - // Return the chunk that contains the given nmethod, NULL otherwise. 2.186 - // Scans the list of chunks backwards, as this method is used to add new 2.187 - // entries, which are typically added in bulk for a single nmethod. 2.188 - G1CodeRootChunk* find(nmethod* method); 2.189 - void free(G1CodeRootChunk* chunk); 2.190 + CodeRootSetTable* _table; 2.191 + CodeRootSetTable* load_acquire_table(); 2.192 2.193 size_t _length; 2.194 - FreeList<G1CodeRootChunk> _list; 2.195 - G1CodeRootChunkManager* _manager; 2.196 + 2.197 + void move_to_large(); 2.198 + void allocate_small_table(); 2.199 2.200 public: 2.201 - // If an instance is initialized with a chunk manager of NULL, use the global 2.202 - // default one. 2.203 - G1CodeRootSet(G1CodeRootChunkManager* manager = NULL); 2.204 + G1CodeRootSet() : _table(NULL), _length(0) {} 2.205 ~G1CodeRootSet(); 2.206 2.207 - static void purge_chunks(size_t keep_ratio); 2.208 + static void purge(); 2.209 2.210 - static size_t free_chunks_static_mem_size(); 2.211 - static size_t free_chunks_mem_size(); 2.212 + static size_t static_mem_size(); 2.213 2.214 - // Search for the code blob from the recently allocated ones to find duplicates more quickly, as this 2.215 - // method is likely to be repeatedly called with the same nmethod. 2.216 void add(nmethod* method); 2.217 2.218 - void remove_lock_free(nmethod* method); 2.219 - nmethod* pop(); 2.220 + bool remove(nmethod* method); 2.221 2.222 + // Safe to call without synchronization, but may return false negatives. 2.223 bool contains(nmethod* method); 2.224 2.225 void clear(); 2.226 2.227 void nmethods_do(CodeBlobClosure* blk) const; 2.228 2.229 - bool is_empty() { return length() == 0; } 2.230 + // Remove all nmethods which no longer contain pointers into our "owner" region 2.231 + void clean(HeapRegion* owner); 2.232 + 2.233 + bool is_empty() { 2.234 + bool empty = length() == 0; 2.235 + assert(empty == (_table == NULL), "is empty only if table is deallocated"); 2.236 + return empty; 2.237 + } 2.238 2.239 // Length in elements 2.240 size_t length() const { return _length; } 2.241 2.242 - // Static data memory size in bytes of this set. 2.243 - static size_t static_mem_size(); 2.244 // Memory size in bytes taken by this set. 2.245 size_t mem_size(); 2.246 2.247 - static void test() PRODUCT_RETURN; 2.248 }; 2.249 2.250 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1CODECACHEREMSET_HPP
3.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Fri Aug 29 13:08:01 2014 +0200 3.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Fri Aug 29 13:12:21 2014 +0200 3.3 @@ -4580,6 +4580,56 @@ 3.4 } 3.5 }; 3.6 3.7 +class G1CodeBlobClosure : public CodeBlobClosure { 3.8 + class HeapRegionGatheringOopClosure : public OopClosure { 3.9 + G1CollectedHeap* _g1h; 3.10 + OopClosure* _work; 3.11 + nmethod* _nm; 3.12 + 3.13 + template <typename T> 3.14 + void do_oop_work(T* p) { 3.15 + _work->do_oop(p); 3.16 + T oop_or_narrowoop = oopDesc::load_heap_oop(p); 3.17 + if (!oopDesc::is_null(oop_or_narrowoop)) { 3.18 + oop o = oopDesc::decode_heap_oop_not_null(oop_or_narrowoop); 3.19 + HeapRegion* hr = _g1h->heap_region_containing_raw(o); 3.20 + assert(!_g1h->obj_in_cs(o) || hr->rem_set()->strong_code_roots_list_contains(_nm), "if o still in CS then evacuation failed and nm must already be in the remset"); 3.21 + hr->add_strong_code_root(_nm); 3.22 + } 3.23 + } 3.24 + 3.25 + public: 3.26 + HeapRegionGatheringOopClosure(OopClosure* oc) : _g1h(G1CollectedHeap::heap()), _work(oc), _nm(NULL) {} 3.27 + 3.28 + void do_oop(oop* o) { 3.29 + do_oop_work(o); 3.30 + } 3.31 + 3.32 + void do_oop(narrowOop* o) { 3.33 + do_oop_work(o); 3.34 + } 3.35 + 3.36 + void set_nm(nmethod* nm) { 3.37 + _nm = nm; 3.38 + } 3.39 + }; 3.40 + 3.41 + HeapRegionGatheringOopClosure _oc; 3.42 +public: 3.43 + G1CodeBlobClosure(OopClosure* oc) : _oc(oc) {} 3.44 + 3.45 + void do_code_blob(CodeBlob* cb) { 3.46 + nmethod* nm = cb->as_nmethod_or_null(); 3.47 + if (nm != NULL) { 3.48 + if (!nm->test_set_oops_do_mark()) { 3.49 + _oc.set_nm(nm); 3.50 + nm->oops_do(&_oc); 3.51 + nm->fix_oop_relocations(); 3.52 + } 3.53 + } 3.54 + } 3.55 +}; 3.56 + 3.57 class G1ParTask : public AbstractGangTask { 3.58 protected: 3.59 G1CollectedHeap* _g1h; 3.60 @@ -4648,22 +4698,6 @@ 3.61 } 3.62 }; 3.63 3.64 - class G1CodeBlobClosure: public CodeBlobClosure { 3.65 - OopClosure* _f; 3.66 - 3.67 - public: 3.68 - G1CodeBlobClosure(OopClosure* f) : _f(f) {} 3.69 - void do_code_blob(CodeBlob* blob) { 3.70 - nmethod* that = blob->as_nmethod_or_null(); 3.71 - if (that != NULL) { 3.72 - if (!that->test_set_oops_do_mark()) { 3.73 - that->oops_do(_f); 3.74 - that->fix_oop_relocations(); 3.75 - } 3.76 - } 3.77 - } 3.78 - }; 3.79 - 3.80 void work(uint worker_id) { 3.81 if (worker_id >= _n_workers) return; // no work needed this round 3.82 3.83 @@ -4854,7 +4888,7 @@ 3.84 g1_policy()->phase_times()->record_satb_filtering_time(worker_i, satb_filtering_ms); 3.85 3.86 // Now scan the complement of the collection set. 3.87 - MarkingCodeBlobClosure scavenge_cs_nmethods(scan_non_heap_weak_roots, CodeBlobToOopClosure::FixRelocations); 3.88 + G1CodeBlobClosure scavenge_cs_nmethods(scan_non_heap_weak_roots); 3.89 3.90 g1_rem_set()->oops_into_collection_set_do(scan_rs, &scavenge_cs_nmethods, worker_i); 3.91 3.92 @@ -5901,12 +5935,6 @@ 3.93 hot_card_cache->reset_hot_cache(); 3.94 hot_card_cache->set_use_cache(true); 3.95 3.96 - // Migrate the strong code roots attached to each region in 3.97 - // the collection set. Ideally we would like to do this 3.98 - // after we have finished the scanning/evacuation of the 3.99 - // strong code roots for a particular heap region. 3.100 - migrate_strong_code_roots(); 3.101 - 3.102 purge_code_root_memory(); 3.103 3.104 if (g1_policy()->during_initial_mark_pause()) { 3.105 @@ -6902,13 +6930,8 @@ 3.106 " starting at "HR_FORMAT, 3.107 _nm, HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()))); 3.108 3.109 - // HeapRegion::add_strong_code_root() avoids adding duplicate 3.110 - // entries but having duplicates is OK since we "mark" nmethods 3.111 - // as visited when we scan the strong code root lists during the GC. 3.112 - hr->add_strong_code_root(_nm); 3.113 - assert(hr->rem_set()->strong_code_roots_list_contains(_nm), 3.114 - err_msg("failed to add code root "PTR_FORMAT" to remembered set of region "HR_FORMAT, 3.115 - _nm, HR_FORMAT_PARAMS(hr))); 3.116 + // HeapRegion::add_strong_code_root_locked() avoids adding duplicate entries. 3.117 + hr->add_strong_code_root_locked(_nm); 3.118 } 3.119 } 3.120 3.121 @@ -6935,9 +6958,6 @@ 3.122 _nm, HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()))); 3.123 3.124 hr->remove_strong_code_root(_nm); 3.125 - assert(!hr->rem_set()->strong_code_roots_list_contains(_nm), 3.126 - err_msg("failed to remove code root "PTR_FORMAT" of region "HR_FORMAT, 3.127 - _nm, HR_FORMAT_PARAMS(hr))); 3.128 } 3.129 } 3.130 3.131 @@ -6965,28 +6985,9 @@ 3.132 nm->oops_do(®_cl, true); 3.133 } 3.134 3.135 -class MigrateCodeRootsHeapRegionClosure: public HeapRegionClosure { 3.136 -public: 3.137 - bool doHeapRegion(HeapRegion *hr) { 3.138 - assert(!hr->isHumongous(), 3.139 - err_msg("humongous region "HR_FORMAT" should not have been added to collection set", 3.140 - HR_FORMAT_PARAMS(hr))); 3.141 - hr->migrate_strong_code_roots(); 3.142 - return false; 3.143 - } 3.144 -}; 3.145 - 3.146 -void G1CollectedHeap::migrate_strong_code_roots() { 3.147 - MigrateCodeRootsHeapRegionClosure cl; 3.148 - double migrate_start = os::elapsedTime(); 3.149 - collection_set_iterate(&cl); 3.150 - double migration_time_ms = (os::elapsedTime() - migrate_start) * 1000.0; 3.151 - g1_policy()->phase_times()->record_strong_code_root_migration_time(migration_time_ms); 3.152 -} 3.153 - 3.154 void G1CollectedHeap::purge_code_root_memory() { 3.155 double purge_start = os::elapsedTime(); 3.156 - G1CodeRootSet::purge_chunks(G1CodeRootsChunkCacheKeepPercent); 3.157 + G1CodeRootSet::purge(); 3.158 double purge_time_ms = (os::elapsedTime() - purge_start) * 1000.0; 3.159 g1_policy()->phase_times()->record_strong_code_root_purge_time(purge_time_ms); 3.160 }
4.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Fri Aug 29 13:08:01 2014 +0200 4.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Fri Aug 29 13:12:21 2014 +0200 4.3 @@ -1,4 +1,4 @@ 4.4 -/* 4.5 + /* 4.6 * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. 4.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4.8 * 4.9 @@ -1633,12 +1633,6 @@ 4.10 // Unregister the given nmethod from the G1 heap 4.11 virtual void unregister_nmethod(nmethod* nm); 4.12 4.13 - // Migrate the nmethods in the code root lists of the regions 4.14 - // in the collection set to regions in to-space. In the event 4.15 - // of an evacuation failure, nmethods that reference objects 4.16 - // that were not successfullly evacuated are not migrated. 4.17 - void migrate_strong_code_roots(); 4.18 - 4.19 // Free up superfluous code root memory. 4.20 void purge_code_root_memory(); 4.21
5.1 --- a/src/share/vm/gc_implementation/g1/g1EvacFailure.hpp Fri Aug 29 13:08:01 2014 +0200 5.2 +++ b/src/share/vm/gc_implementation/g1/g1EvacFailure.hpp Fri Aug 29 13:12:21 2014 +0200 5.3 @@ -217,6 +217,8 @@ 5.4 _update_rset_cl->set_region(hr); 5.5 hr->object_iterate(&rspc); 5.6 5.7 + hr->rem_set()->clean_strong_code_roots(hr); 5.8 + 5.9 hr->note_self_forwarding_removal_end(during_initial_mark, 5.10 during_conc_mark, 5.11 rspc.marked_bytes());
6.1 --- a/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp Fri Aug 29 13:08:01 2014 +0200 6.2 +++ b/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp Fri Aug 29 13:12:21 2014 +0200 6.3 @@ -274,9 +274,6 @@ 6.4 // Now subtract the time taken to fix up roots in generated code 6.5 misc_time_ms += _cur_collection_code_root_fixup_time_ms; 6.6 6.7 - // Strong code root migration time 6.8 - misc_time_ms += _cur_strong_code_root_migration_time_ms; 6.9 - 6.10 // Strong code root purge time 6.11 misc_time_ms += _cur_strong_code_root_purge_time_ms; 6.12 6.13 @@ -327,7 +324,6 @@ 6.14 _last_obj_copy_times_ms.print(1, "Object Copy (ms)"); 6.15 } 6.16 print_stats(1, "Code Root Fixup", _cur_collection_code_root_fixup_time_ms); 6.17 - print_stats(1, "Code Root Migration", _cur_strong_code_root_migration_time_ms); 6.18 print_stats(1, "Code Root Purge", _cur_strong_code_root_purge_time_ms); 6.19 if (G1StringDedup::is_enabled()) { 6.20 print_stats(1, "String Dedup Fixup", _cur_string_dedup_fixup_time_ms, _active_gc_threads);
7.1 --- a/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp Fri Aug 29 13:08:01 2014 +0200 7.2 +++ b/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp Fri Aug 29 13:12:21 2014 +0200 7.3 @@ -129,7 +129,6 @@ 7.4 7.5 double _cur_collection_par_time_ms; 7.6 double _cur_collection_code_root_fixup_time_ms; 7.7 - double _cur_strong_code_root_migration_time_ms; 7.8 double _cur_strong_code_root_purge_time_ms; 7.9 7.10 double _cur_evac_fail_recalc_used; 7.11 @@ -233,10 +232,6 @@ 7.12 _cur_collection_code_root_fixup_time_ms = ms; 7.13 } 7.14 7.15 - void record_strong_code_root_migration_time(double ms) { 7.16 - _cur_strong_code_root_migration_time_ms = ms; 7.17 - } 7.18 - 7.19 void record_strong_code_root_purge_time(double ms) { 7.20 _cur_strong_code_root_purge_time_ms = ms; 7.21 }
8.1 --- a/src/share/vm/gc_implementation/g1/g1RemSet.cpp Fri Aug 29 13:08:01 2014 +0200 8.2 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.cpp Fri Aug 29 13:12:21 2014 +0200 8.3 @@ -109,7 +109,7 @@ 8.4 G1CollectedHeap* _g1h; 8.5 8.6 OopsInHeapRegionClosure* _oc; 8.7 - CodeBlobToOopClosure* _code_root_cl; 8.8 + CodeBlobClosure* _code_root_cl; 8.9 8.10 G1BlockOffsetSharedArray* _bot_shared; 8.11 G1SATBCardTableModRefBS *_ct_bs; 8.12 @@ -121,7 +121,7 @@ 8.13 8.14 public: 8.15 ScanRSClosure(OopsInHeapRegionClosure* oc, 8.16 - CodeBlobToOopClosure* code_root_cl, 8.17 + CodeBlobClosure* code_root_cl, 8.18 uint worker_i) : 8.19 _oc(oc), 8.20 _code_root_cl(code_root_cl), 8.21 @@ -241,7 +241,7 @@ 8.22 }; 8.23 8.24 void G1RemSet::scanRS(OopsInHeapRegionClosure* oc, 8.25 - CodeBlobToOopClosure* code_root_cl, 8.26 + CodeBlobClosure* code_root_cl, 8.27 uint worker_i) { 8.28 double rs_time_start = os::elapsedTime(); 8.29 HeapRegion *startRegion = _g1->start_cset_region_for_worker(worker_i); 8.30 @@ -320,7 +320,7 @@ 8.31 } 8.32 8.33 void G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc, 8.34 - CodeBlobToOopClosure* code_root_cl, 8.35 + CodeBlobClosure* code_root_cl, 8.36 uint worker_i) { 8.37 #if CARD_REPEAT_HISTO 8.38 ct_freq_update_histo_and_reset();
9.1 --- a/src/share/vm/gc_implementation/g1/g1RemSet.hpp Fri Aug 29 13:08:01 2014 +0200 9.2 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.hpp Fri Aug 29 13:12:21 2014 +0200 9.3 @@ -96,7 +96,7 @@ 9.4 // the "i" passed to the calling thread's work(i) function. 9.5 // In the sequential case this param will be ignored. 9.6 void oops_into_collection_set_do(OopsInHeapRegionClosure* blk, 9.7 - CodeBlobToOopClosure* code_root_cl, 9.8 + CodeBlobClosure* code_root_cl, 9.9 uint worker_i); 9.10 9.11 // Prepare for and cleanup after an oops_into_collection_set_do 9.12 @@ -108,7 +108,7 @@ 9.13 void cleanup_after_oops_into_collection_set_do(); 9.14 9.15 void scanRS(OopsInHeapRegionClosure* oc, 9.16 - CodeBlobToOopClosure* code_root_cl, 9.17 + CodeBlobClosure* code_root_cl, 9.18 uint worker_i); 9.19 9.20 void updateRS(DirtyCardQueue* into_cset_dcq, uint worker_i);
10.1 --- a/src/share/vm/gc_implementation/g1/g1RemSetSummary.cpp Fri Aug 29 13:08:01 2014 +0200 10.2 +++ b/src/share/vm/gc_implementation/g1/g1RemSetSummary.cpp Fri Aug 29 13:12:21 2014 +0200 10.3 @@ -253,6 +253,7 @@ 10.4 size_t occupied_cards = hrrs->occupied(); 10.5 size_t code_root_mem_sz = hrrs->strong_code_roots_mem_size(); 10.6 if (code_root_mem_sz > max_code_root_mem_sz()) { 10.7 + _max_code_root_mem_sz = code_root_mem_sz; 10.8 _max_code_root_mem_sz_region = r; 10.9 } 10.10 size_t code_root_elems = hrrs->strong_code_roots_list_length();
11.1 --- a/src/share/vm/gc_implementation/g1/g1_globals.hpp Fri Aug 29 13:08:01 2014 +0200 11.2 +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp Fri Aug 29 13:12:21 2014 +0200 11.3 @@ -285,10 +285,6 @@ 11.4 product(uintx, G1MixedGCCountTarget, 8, \ 11.5 "The target number of mixed GCs after a marking cycle.") \ 11.6 \ 11.7 - experimental(uintx, G1CodeRootsChunkCacheKeepPercent, 10, \ 11.8 - "The amount of code root chunks that should be kept at most " \ 11.9 - "as percentage of already allocated.") \ 11.10 - \ 11.11 experimental(bool, G1ReclaimDeadHumongousObjectsAtYoungGC, true, \ 11.12 "Try to reclaim dead large objects at every young GC.") \ 11.13 \
12.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.cpp Fri Aug 29 13:08:01 2014 +0200 12.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp Fri Aug 29 13:12:21 2014 +0200 12.3 @@ -549,21 +549,17 @@ 12.4 hrrs->add_strong_code_root(nm); 12.5 } 12.6 12.7 +void HeapRegion::add_strong_code_root_locked(nmethod* nm) { 12.8 + assert_locked_or_safepoint(CodeCache_lock); 12.9 + HeapRegionRemSet* hrrs = rem_set(); 12.10 + hrrs->add_strong_code_root_locked(nm); 12.11 +} 12.12 + 12.13 void HeapRegion::remove_strong_code_root(nmethod* nm) { 12.14 HeapRegionRemSet* hrrs = rem_set(); 12.15 hrrs->remove_strong_code_root(nm); 12.16 } 12.17 12.18 -void HeapRegion::migrate_strong_code_roots() { 12.19 - assert(in_collection_set(), "only collection set regions"); 12.20 - assert(!isHumongous(), 12.21 - err_msg("humongous region "HR_FORMAT" should not have been added to collection set", 12.22 - HR_FORMAT_PARAMS(this))); 12.23 - 12.24 - HeapRegionRemSet* hrrs = rem_set(); 12.25 - hrrs->migrate_strong_code_roots(); 12.26 -} 12.27 - 12.28 void HeapRegion::strong_code_roots_do(CodeBlobClosure* blk) const { 12.29 HeapRegionRemSet* hrrs = rem_set(); 12.30 hrrs->strong_code_roots_do(blk);
13.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.hpp Fri Aug 29 13:08:01 2014 +0200 13.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp Fri Aug 29 13:12:21 2014 +0200 13.3 @@ -772,14 +772,9 @@ 13.4 // Routines for managing a list of code roots (attached to the 13.5 // this region's RSet) that point into this heap region. 13.6 void add_strong_code_root(nmethod* nm); 13.7 + void add_strong_code_root_locked(nmethod* nm); 13.8 void remove_strong_code_root(nmethod* nm); 13.9 13.10 - // During a collection, migrate the successfully evacuated 13.11 - // strong code roots that referenced into this region to the 13.12 - // new regions that they now point into. Unsuccessfully 13.13 - // evacuated code roots are not migrated. 13.14 - void migrate_strong_code_roots(); 13.15 - 13.16 // Applies blk->do_code_blob() to each of the entries in 13.17 // the strong code roots list for this region 13.18 void strong_code_roots_do(CodeBlobClosure* blk) const;
14.1 --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp Fri Aug 29 13:08:01 2014 +0200 14.2 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp Fri Aug 29 13:12:21 2014 +0200 14.3 @@ -923,9 +923,25 @@ 14.4 } 14.5 14.6 // Code roots support 14.7 +// 14.8 +// The code root set is protected by two separate locking schemes 14.9 +// When at safepoint the per-hrrs lock must be held during modifications 14.10 +// except when doing a full gc. 14.11 +// When not at safepoint the CodeCache_lock must be held during modifications. 14.12 +// When concurrent readers access the contains() function 14.13 +// (during the evacuation phase) no removals are allowed. 14.14 14.15 void HeapRegionRemSet::add_strong_code_root(nmethod* nm) { 14.16 assert(nm != NULL, "sanity"); 14.17 + // Optimistic unlocked contains-check 14.18 + if (!_code_roots.contains(nm)) { 14.19 + MutexLockerEx ml(&_m, Mutex::_no_safepoint_check_flag); 14.20 + add_strong_code_root_locked(nm); 14.21 + } 14.22 +} 14.23 + 14.24 +void HeapRegionRemSet::add_strong_code_root_locked(nmethod* nm) { 14.25 + assert(nm != NULL, "sanity"); 14.26 _code_roots.add(nm); 14.27 } 14.28 14.29 @@ -933,96 +949,19 @@ 14.30 assert(nm != NULL, "sanity"); 14.31 assert_locked_or_safepoint(CodeCache_lock); 14.32 14.33 - _code_roots.remove_lock_free(nm); 14.34 + MutexLockerEx ml(CodeCache_lock->owned_by_self() ? NULL : &_m, Mutex::_no_safepoint_check_flag); 14.35 + _code_roots.remove(nm); 14.36 14.37 // Check that there were no duplicates 14.38 guarantee(!_code_roots.contains(nm), "duplicate entry found"); 14.39 } 14.40 14.41 -class NMethodMigrationOopClosure : public OopClosure { 14.42 - G1CollectedHeap* _g1h; 14.43 - HeapRegion* _from; 14.44 - nmethod* _nm; 14.45 - 14.46 - uint _num_self_forwarded; 14.47 - 14.48 - template <class T> void do_oop_work(T* p) { 14.49 - T heap_oop = oopDesc::load_heap_oop(p); 14.50 - if (!oopDesc::is_null(heap_oop)) { 14.51 - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); 14.52 - if (_from->is_in(obj)) { 14.53 - // Reference still points into the source region. 14.54 - // Since roots are immediately evacuated this means that 14.55 - // we must have self forwarded the object 14.56 - assert(obj->is_forwarded(), 14.57 - err_msg("code roots should be immediately evacuated. " 14.58 - "Ref: "PTR_FORMAT", " 14.59 - "Obj: "PTR_FORMAT", " 14.60 - "Region: "HR_FORMAT, 14.61 - p, (void*) obj, HR_FORMAT_PARAMS(_from))); 14.62 - assert(obj->forwardee() == obj, 14.63 - err_msg("not self forwarded? obj = "PTR_FORMAT, (void*)obj)); 14.64 - 14.65 - // The object has been self forwarded. 14.66 - // Note, if we're during an initial mark pause, there is 14.67 - // no need to explicitly mark object. It will be marked 14.68 - // during the regular evacuation failure handling code. 14.69 - _num_self_forwarded++; 14.70 - } else { 14.71 - // The reference points into a promotion or to-space region 14.72 - HeapRegion* to = _g1h->heap_region_containing(obj); 14.73 - to->rem_set()->add_strong_code_root(_nm); 14.74 - } 14.75 - } 14.76 - } 14.77 - 14.78 -public: 14.79 - NMethodMigrationOopClosure(G1CollectedHeap* g1h, HeapRegion* from, nmethod* nm): 14.80 - _g1h(g1h), _from(from), _nm(nm), _num_self_forwarded(0) {} 14.81 - 14.82 - void do_oop(narrowOop* p) { do_oop_work(p); } 14.83 - void do_oop(oop* p) { do_oop_work(p); } 14.84 - 14.85 - uint retain() { return _num_self_forwarded > 0; } 14.86 -}; 14.87 - 14.88 -void HeapRegionRemSet::migrate_strong_code_roots() { 14.89 - assert(hr()->in_collection_set(), "only collection set regions"); 14.90 - assert(!hr()->isHumongous(), 14.91 - err_msg("humongous region "HR_FORMAT" should not have been added to the collection set", 14.92 - HR_FORMAT_PARAMS(hr()))); 14.93 - 14.94 - ResourceMark rm; 14.95 - 14.96 - // List of code blobs to retain for this region 14.97 - GrowableArray<nmethod*> to_be_retained(10); 14.98 - G1CollectedHeap* g1h = G1CollectedHeap::heap(); 14.99 - 14.100 - while (!_code_roots.is_empty()) { 14.101 - nmethod *nm = _code_roots.pop(); 14.102 - if (nm != NULL) { 14.103 - NMethodMigrationOopClosure oop_cl(g1h, hr(), nm); 14.104 - nm->oops_do(&oop_cl); 14.105 - if (oop_cl.retain()) { 14.106 - to_be_retained.push(nm); 14.107 - } 14.108 - } 14.109 - } 14.110 - 14.111 - // Now push any code roots we need to retain 14.112 - assert(to_be_retained.is_empty() || hr()->evacuation_failed(), 14.113 - "Retained nmethod list must be empty or " 14.114 - "evacuation of this region failed"); 14.115 - 14.116 - while (to_be_retained.is_nonempty()) { 14.117 - nmethod* nm = to_be_retained.pop(); 14.118 - assert(nm != NULL, "sanity"); 14.119 - add_strong_code_root(nm); 14.120 - } 14.121 +void HeapRegionRemSet::strong_code_roots_do(CodeBlobClosure* blk) const { 14.122 + _code_roots.nmethods_do(blk); 14.123 } 14.124 14.125 -void HeapRegionRemSet::strong_code_roots_do(CodeBlobClosure* blk) const { 14.126 - _code_roots.nmethods_do(blk); 14.127 +void HeapRegionRemSet::clean_strong_code_roots(HeapRegion* hr) { 14.128 + _code_roots.clean(hr); 14.129 } 14.130 14.131 size_t HeapRegionRemSet::strong_code_roots_mem_size() {
15.1 --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp Fri Aug 29 13:08:01 2014 +0200 15.2 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp Fri Aug 29 13:12:21 2014 +0200 15.3 @@ -353,13 +353,13 @@ 15.4 // Returns the memory occupancy of all static data structures associated 15.5 // with remembered sets. 15.6 static size_t static_mem_size() { 15.7 - return OtherRegionsTable::static_mem_size() + G1CodeRootSet::free_chunks_static_mem_size(); 15.8 + return OtherRegionsTable::static_mem_size() + G1CodeRootSet::static_mem_size(); 15.9 } 15.10 15.11 // Returns the memory occupancy of all free_list data structures associated 15.12 // with remembered sets. 15.13 static size_t fl_mem_size() { 15.14 - return OtherRegionsTable::fl_mem_size() + G1CodeRootSet::free_chunks_mem_size(); 15.15 + return OtherRegionsTable::fl_mem_size(); 15.16 } 15.17 15.18 bool contains_reference(OopOrNarrowOopStar from) const { 15.19 @@ -369,18 +369,15 @@ 15.20 // Routines for managing the list of code roots that point into 15.21 // the heap region that owns this RSet. 15.22 void add_strong_code_root(nmethod* nm); 15.23 + void add_strong_code_root_locked(nmethod* nm); 15.24 void remove_strong_code_root(nmethod* nm); 15.25 15.26 - // During a collection, migrate the successfully evacuated strong 15.27 - // code roots that referenced into the region that owns this RSet 15.28 - // to the RSets of the new regions that they now point into. 15.29 - // Unsuccessfully evacuated code roots are not migrated. 15.30 - void migrate_strong_code_roots(); 15.31 - 15.32 // Applies blk->do_code_blob() to each of the entries in 15.33 // the strong code roots list 15.34 void strong_code_roots_do(CodeBlobClosure* blk) const; 15.35 15.36 + void clean_strong_code_roots(HeapRegion* hr); 15.37 + 15.38 // Returns the number of elements in the strong code roots list 15.39 size_t strong_code_roots_list_length() const { 15.40 return _code_roots.length();
16.1 --- a/src/share/vm/memory/freeList.cpp Fri Aug 29 13:08:01 2014 +0200 16.2 +++ b/src/share/vm/memory/freeList.cpp Fri Aug 29 13:12:21 2014 +0200 16.3 @@ -34,7 +34,6 @@ 16.4 16.5 #if INCLUDE_ALL_GCS 16.6 #include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" 16.7 -#include "gc_implementation/g1/g1CodeCacheRemSet.hpp" 16.8 #endif // INCLUDE_ALL_GCS 16.9 16.10 // Free list. A FreeList is used to access a linked list of chunks 16.11 @@ -333,5 +332,4 @@ 16.12 template class FreeList<Metachunk>; 16.13 #if INCLUDE_ALL_GCS 16.14 template class FreeList<FreeChunk>; 16.15 -template class FreeList<G1CodeRootChunk>; 16.16 #endif // INCLUDE_ALL_GCS
17.1 --- a/src/share/vm/utilities/hashtable.cpp Fri Aug 29 13:08:01 2014 +0200 17.2 +++ b/src/share/vm/utilities/hashtable.cpp Fri Aug 29 13:12:21 2014 +0200 17.3 @@ -353,6 +353,11 @@ 17.4 17.5 #endif 17.6 // Explicitly instantiate these types 17.7 +#if INCLUDE_ALL_GCS 17.8 +template class Hashtable<nmethod*, mtGC>; 17.9 +template class HashtableEntry<nmethod*, mtGC>; 17.10 +template class BasicHashtable<mtGC>; 17.11 +#endif 17.12 template class Hashtable<ConstantPool*, mtClass>; 17.13 template class RehashableHashtable<Symbol*, mtSymbol>; 17.14 template class RehashableHashtable<oopDesc*, mtSymbol>;
