Fri, 01 Mar 2013 10:19:29 -0800
8011268: NPG: Free unused VirtualSpaceNodes
Reviewed-by: mgerdin, coleenp, johnc
1.1 --- a/src/share/vm/classfile/classLoaderData.cpp Thu Apr 18 06:50:35 2013 +0200 1.2 +++ b/src/share/vm/classfile/classLoaderData.cpp Fri Mar 01 10:19:29 2013 -0800 1.3 @@ -686,6 +686,7 @@ 1.4 next = purge_me->next(); 1.5 delete purge_me; 1.6 } 1.7 + Metaspace::purge(); 1.8 } 1.9 1.10 // CDS support
2.1 --- a/src/share/vm/memory/metachunk.cpp Thu Apr 18 06:50:35 2013 +0200 2.2 +++ b/src/share/vm/memory/metachunk.cpp Fri Mar 01 10:19:29 2013 -0800 2.3 @@ -28,6 +28,7 @@ 2.4 #include "utilities/copy.hpp" 2.5 #include "utilities/debug.hpp" 2.6 2.7 +class VirtualSpaceNode; 2.8 // 2.9 // Future modification 2.10 // 2.11 @@ -45,27 +46,30 @@ 2.12 2.13 // Metachunk methods 2.14 2.15 -Metachunk* Metachunk::initialize(MetaWord* ptr, size_t word_size) { 2.16 - // Set bottom, top, and end. Allow space for the Metachunk itself 2.17 - Metachunk* chunk = (Metachunk*) ptr; 2.18 - 2.19 - MetaWord* chunk_bottom = ptr + _overhead; 2.20 - chunk->set_bottom(ptr); 2.21 - chunk->set_top(chunk_bottom); 2.22 - MetaWord* chunk_end = ptr + word_size; 2.23 - assert(chunk_end > chunk_bottom, "Chunk must be too small"); 2.24 - chunk->set_end(chunk_end); 2.25 - chunk->set_next(NULL); 2.26 - chunk->set_prev(NULL); 2.27 - chunk->set_word_size(word_size); 2.28 +Metachunk::Metachunk(size_t word_size, 2.29 + VirtualSpaceNode* container) : 2.30 + _word_size(word_size), 2.31 + _bottom(NULL), 2.32 + _end(NULL), 2.33 + _top(NULL), 2.34 + _next(NULL), 2.35 + _prev(NULL), 2.36 + _container(container) 2.37 +{ 2.38 + _bottom = (MetaWord*)this; 2.39 + _top = (MetaWord*)this + _overhead; 2.40 + _end = (MetaWord*)this + word_size; 2.41 #ifdef ASSERT 2.42 - size_t data_word_size = pointer_delta(chunk_end, chunk_bottom, sizeof(MetaWord)); 2.43 - Copy::fill_to_words((HeapWord*) chunk_bottom, data_word_size, metadata_chunk_initialize); 2.44 + set_is_free(false); 2.45 + size_t data_word_size = pointer_delta(end(), 2.46 + top(), 2.47 + sizeof(MetaWord)); 2.48 + Copy::fill_to_words((HeapWord*) top(), 2.49 + data_word_size, 2.50 + metadata_chunk_initialize); 2.51 #endif 2.52 - return chunk; 2.53 } 2.54 2.55 - 2.56 MetaWord* Metachunk::allocate(size_t word_size) { 2.57 MetaWord* result = NULL; 2.58 // If available, bump the pointer to allocate.
3.1 --- a/src/share/vm/memory/metachunk.hpp Thu Apr 18 06:50:35 2013 +0200 3.2 +++ b/src/share/vm/memory/metachunk.hpp Fri Mar 01 10:19:29 2013 -0800 3.3 @@ -41,10 +41,13 @@ 3.4 // | | | | 3.5 // +--------------+ <- bottom ---+ ---+ 3.6 3.7 +class VirtualSpaceNode; 3.8 + 3.9 class Metachunk VALUE_OBJ_CLASS_SPEC { 3.10 // link to support lists of chunks 3.11 Metachunk* _next; 3.12 Metachunk* _prev; 3.13 + VirtualSpaceNode* _container; 3.14 3.15 MetaWord* _bottom; 3.16 MetaWord* _end; 3.17 @@ -61,29 +64,20 @@ 3.18 // the space. 3.19 static size_t _overhead; 3.20 3.21 - void set_bottom(MetaWord* v) { _bottom = v; } 3.22 - void set_end(MetaWord* v) { _end = v; } 3.23 - void set_top(MetaWord* v) { _top = v; } 3.24 - void set_word_size(size_t v) { _word_size = v; } 3.25 public: 3.26 -#ifdef ASSERT 3.27 - Metachunk() : _bottom(NULL), _end(NULL), _top(NULL), _is_free(false), 3.28 - _next(NULL), _prev(NULL) {} 3.29 -#else 3.30 - Metachunk() : _bottom(NULL), _end(NULL), _top(NULL), 3.31 - _next(NULL), _prev(NULL) {} 3.32 -#endif 3.33 + Metachunk(size_t word_size , VirtualSpaceNode* container); 3.34 3.35 // Used to add a Metachunk to a list of Metachunks 3.36 void set_next(Metachunk* v) { _next = v; assert(v != this, "Boom");} 3.37 void set_prev(Metachunk* v) { _prev = v; assert(v != this, "Boom");} 3.38 + void set_container(VirtualSpaceNode* v) { _container = v; } 3.39 3.40 MetaWord* allocate(size_t word_size); 3.41 - static Metachunk* initialize(MetaWord* ptr, size_t word_size); 3.42 3.43 // Accessors 3.44 Metachunk* next() const { return _next; } 3.45 Metachunk* prev() const { return _prev; } 3.46 + VirtualSpaceNode* container() const { return _container; } 3.47 MetaWord* bottom() const { return _bottom; } 3.48 MetaWord* end() const { return _end; } 3.49 MetaWord* top() const { return _top; }
4.1 --- a/src/share/vm/memory/metaspace.cpp Thu Apr 18 06:50:35 2013 +0200 4.2 +++ b/src/share/vm/memory/metaspace.cpp Fri Mar 01 10:19:29 2013 -0800 4.3 @@ -112,6 +112,7 @@ 4.4 class ChunkManager VALUE_OBJ_CLASS_SPEC { 4.5 4.6 // Free list of chunks of different sizes. 4.7 + // SpecializedChunk 4.8 // SmallChunk 4.9 // MediumChunk 4.10 // HumongousChunk 4.11 @@ -165,6 +166,10 @@ 4.12 // for special, small, medium, and humongous chunks. 4.13 static ChunkIndex list_index(size_t size); 4.14 4.15 + // Remove the chunk from its freelist. It is 4.16 + // expected to be on one of the _free_chunks[] lists. 4.17 + void remove_chunk(Metachunk* chunk); 4.18 + 4.19 // Add the simple linked list of chunks to the freelist of chunks 4.20 // of type index. 4.21 void return_chunks(ChunkIndex index, Metachunk* chunks); 4.22 @@ -255,6 +260,8 @@ 4.23 ReservedSpace _rs; 4.24 VirtualSpace _virtual_space; 4.25 MetaWord* _top; 4.26 + // count of chunks contained in this VirtualSpace 4.27 + uintx _container_count; 4.28 4.29 // Convenience functions for logical bottom and end 4.30 MetaWord* bottom() const { return (MetaWord*) _virtual_space.low(); } 4.31 @@ -264,10 +271,19 @@ 4.32 char* low() const { return virtual_space()->low(); } 4.33 char* high() const { return virtual_space()->high(); } 4.34 4.35 + // The first Metachunk will be allocated at the bottom of the 4.36 + // VirtualSpace 4.37 + Metachunk* first_chunk() { return (Metachunk*) bottom(); } 4.38 + 4.39 + void inc_container_count(); 4.40 +#ifdef ASSERT 4.41 + uint container_count_slow(); 4.42 +#endif 4.43 + 4.44 public: 4.45 4.46 VirtualSpaceNode(size_t byte_size); 4.47 - VirtualSpaceNode(ReservedSpace rs) : _top(NULL), _next(NULL), _rs(rs) {} 4.48 + VirtualSpaceNode(ReservedSpace rs) : _top(NULL), _next(NULL), _rs(rs), _container_count(0) {} 4.49 ~VirtualSpaceNode(); 4.50 4.51 // address of next available space in _virtual_space; 4.52 @@ -288,6 +304,12 @@ 4.53 MetaWord* top() const { return _top; } 4.54 void inc_top(size_t word_size) { _top += word_size; } 4.55 4.56 + uintx container_count() { return _container_count; } 4.57 + void dec_container_count(); 4.58 +#ifdef ASSERT 4.59 + void verify_container_count(); 4.60 +#endif 4.61 + 4.62 // used and capacity in this single entry in the list 4.63 size_t used_words_in_vs() const; 4.64 size_t capacity_words_in_vs() const; 4.65 @@ -306,6 +328,10 @@ 4.66 bool expand_by(size_t words, bool pre_touch = false); 4.67 bool shrink_by(size_t words); 4.68 4.69 + // In preparation for deleting this node, remove all the chunks 4.70 + // in the node from any freelist. 4.71 + void purge(ChunkManager* chunk_manager); 4.72 + 4.73 #ifdef ASSERT 4.74 // Debug support 4.75 static void verify_virtual_space_total(); 4.76 @@ -317,7 +343,7 @@ 4.77 }; 4.78 4.79 // byte_size is the size of the associated virtualspace. 4.80 -VirtualSpaceNode::VirtualSpaceNode(size_t byte_size) : _top(NULL), _next(NULL), _rs(0) { 4.81 +VirtualSpaceNode::VirtualSpaceNode(size_t byte_size) : _top(NULL), _next(NULL), _rs(0), _container_count(0) { 4.82 // align up to vm allocation granularity 4.83 byte_size = align_size_up(byte_size, os::vm_allocation_granularity()); 4.84 4.85 @@ -341,6 +367,39 @@ 4.86 MemTracker::record_virtual_memory_type((address)_rs.base(), mtClass); 4.87 } 4.88 4.89 +void VirtualSpaceNode::purge(ChunkManager* chunk_manager) { 4.90 + Metachunk* chunk = first_chunk(); 4.91 + Metachunk* invalid_chunk = (Metachunk*) top(); 4.92 + while (chunk < invalid_chunk ) { 4.93 + assert(chunk->is_free(), "Should be marked free"); 4.94 + MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); 4.95 + chunk_manager->remove_chunk(chunk); 4.96 + assert(chunk->next() == NULL && 4.97 + chunk->prev() == NULL, 4.98 + "Was not removed from its list"); 4.99 + chunk = (Metachunk*) next; 4.100 + } 4.101 +} 4.102 + 4.103 +#ifdef ASSERT 4.104 +uint VirtualSpaceNode::container_count_slow() { 4.105 + uint count = 0; 4.106 + Metachunk* chunk = first_chunk(); 4.107 + Metachunk* invalid_chunk = (Metachunk*) top(); 4.108 + while (chunk < invalid_chunk ) { 4.109 + MetaWord* next = ((MetaWord*)chunk) + chunk->word_size(); 4.110 + // Don't count the chunks on the free lists. Those are 4.111 + // still part of the VirtualSpaceNode but not currently 4.112 + // counted. 4.113 + if (!chunk->is_free()) { 4.114 + count++; 4.115 + } 4.116 + chunk = (Metachunk*) next; 4.117 + } 4.118 + return count; 4.119 +} 4.120 +#endif 4.121 + 4.122 // List of VirtualSpaces for metadata allocation. 4.123 // It has a _next link for singly linked list and a MemRegion 4.124 // for total space in the VirtualSpace. 4.125 @@ -410,14 +469,14 @@ 4.126 void initialize(size_t word_size); 4.127 4.128 size_t virtual_space_total() { return _virtual_space_total; } 4.129 - void inc_virtual_space_total(size_t v) { 4.130 - Atomic::add_ptr(v, &_virtual_space_total); 4.131 - } 4.132 - 4.133 - size_t virtual_space_count() { return _virtual_space_count; } 4.134 - void inc_virtual_space_count() { 4.135 - Atomic::inc_ptr(&_virtual_space_count); 4.136 - } 4.137 + 4.138 + void inc_virtual_space_total(size_t v); 4.139 + void dec_virtual_space_total(size_t v); 4.140 + void inc_virtual_space_count(); 4.141 + void dec_virtual_space_count(); 4.142 + 4.143 + // Unlink empty VirtualSpaceNodes and free it. 4.144 + void purge(); 4.145 4.146 // Used and capacity in the entire list of virtual spaces. 4.147 // These are global values shared by all Metaspaces 4.148 @@ -641,6 +700,28 @@ 4.149 SpaceManager::_expand_lock_name, 4.150 Mutex::_allow_vm_block_flag); 4.151 4.152 +void VirtualSpaceNode::inc_container_count() { 4.153 + assert_lock_strong(SpaceManager::expand_lock()); 4.154 + _container_count++; 4.155 + assert(_container_count == container_count_slow(), 4.156 + err_msg("Inconsistency in countainer_count _container_count " SIZE_FORMAT 4.157 + "container_count_slow() " SIZE_FORMAT, 4.158 + _container_count, container_count_slow())); 4.159 +} 4.160 + 4.161 +void VirtualSpaceNode::dec_container_count() { 4.162 + assert_lock_strong(SpaceManager::expand_lock()); 4.163 + _container_count--; 4.164 +} 4.165 + 4.166 +#ifdef ASSERT 4.167 +void VirtualSpaceNode::verify_container_count() { 4.168 + assert(_container_count == container_count_slow(), 4.169 + err_msg("Inconsistency in countainer_count _container_count " SIZE_FORMAT 4.170 + "container_count_slow() " SIZE_FORMAT, _container_count, container_count_slow())); 4.171 +} 4.172 +#endif 4.173 + 4.174 // BlockFreelist methods 4.175 4.176 BlockFreelist::BlockFreelist() : _dictionary(NULL) {} 4.177 @@ -701,6 +782,10 @@ 4.178 4.179 VirtualSpaceNode::~VirtualSpaceNode() { 4.180 _rs.release(); 4.181 +#ifdef ASSERT 4.182 + size_t word_size = sizeof(*this) / BytesPerWord; 4.183 + Copy::fill_to_words((HeapWord*) this, word_size, 0xf1f1f1f1); 4.184 +#endif 4.185 } 4.186 4.187 size_t VirtualSpaceNode::used_words_in_vs() const { 4.188 @@ -733,8 +818,8 @@ 4.189 // Take the space (bump top on the current virtual space). 4.190 inc_top(chunk_word_size); 4.191 4.192 - // Point the chunk at the space 4.193 - Metachunk* result = Metachunk::initialize(chunk_limit, chunk_word_size); 4.194 + // Initialize the chunk 4.195 + Metachunk* result = ::new (chunk_limit) Metachunk(chunk_word_size, this); 4.196 return result; 4.197 } 4.198 4.199 @@ -762,9 +847,11 @@ 4.200 4.201 Metachunk* VirtualSpaceNode::get_chunk_vs(size_t chunk_word_size) { 4.202 assert_lock_strong(SpaceManager::expand_lock()); 4.203 - Metachunk* result = NULL; 4.204 - 4.205 - return take_from_committed(chunk_word_size); 4.206 + Metachunk* result = take_from_committed(chunk_word_size); 4.207 + if (result != NULL) { 4.208 + inc_container_count(); 4.209 + } 4.210 + return result; 4.211 } 4.212 4.213 Metachunk* VirtualSpaceNode::get_chunk_vs_with_expand(size_t chunk_word_size) { 4.214 @@ -843,6 +930,83 @@ 4.215 } 4.216 } 4.217 4.218 +void VirtualSpaceList::inc_virtual_space_total(size_t v) { 4.219 + assert_lock_strong(SpaceManager::expand_lock()); 4.220 + _virtual_space_total = _virtual_space_total + v; 4.221 +} 4.222 +void VirtualSpaceList::dec_virtual_space_total(size_t v) { 4.223 + assert_lock_strong(SpaceManager::expand_lock()); 4.224 + _virtual_space_total = _virtual_space_total - v; 4.225 +} 4.226 + 4.227 +void VirtualSpaceList::inc_virtual_space_count() { 4.228 + assert_lock_strong(SpaceManager::expand_lock()); 4.229 + _virtual_space_count++; 4.230 +} 4.231 +void VirtualSpaceList::dec_virtual_space_count() { 4.232 + assert_lock_strong(SpaceManager::expand_lock()); 4.233 + _virtual_space_count--; 4.234 +} 4.235 + 4.236 +void ChunkManager::remove_chunk(Metachunk* chunk) { 4.237 + size_t word_size = chunk->word_size(); 4.238 + ChunkIndex index = list_index(word_size); 4.239 + if (index != HumongousIndex) { 4.240 + free_chunks(index)->remove_chunk(chunk); 4.241 + } else { 4.242 + humongous_dictionary()->remove_chunk(chunk); 4.243 + } 4.244 + 4.245 + // Chunk is being removed from the chunks free list. 4.246 + dec_free_chunks_total(chunk->capacity_word_size()); 4.247 +} 4.248 + 4.249 +// Walk the list of VirtualSpaceNodes and delete 4.250 +// nodes with a 0 container_count. Remove Metachunks in 4.251 +// the node from their respective freelists. 4.252 +void VirtualSpaceList::purge() { 4.253 + assert_lock_strong(SpaceManager::expand_lock()); 4.254 + // Don't use a VirtualSpaceListIterator because this 4.255 + // list is being changed and a straightforward use of an iterator is not safe. 4.256 + VirtualSpaceNode* purged_vsl = NULL; 4.257 + VirtualSpaceNode* prev_vsl = virtual_space_list(); 4.258 + VirtualSpaceNode* next_vsl = prev_vsl; 4.259 + while (next_vsl != NULL) { 4.260 + VirtualSpaceNode* vsl = next_vsl; 4.261 + next_vsl = vsl->next(); 4.262 + // Don't free the current virtual space since it will likely 4.263 + // be needed soon. 4.264 + if (vsl->container_count() == 0 && vsl != current_virtual_space()) { 4.265 + // Unlink it from the list 4.266 + if (prev_vsl == vsl) { 4.267 + // This is the case of the current note being the first note. 4.268 + assert(vsl == virtual_space_list(), "Expected to be the first note"); 4.269 + set_virtual_space_list(vsl->next()); 4.270 + } else { 4.271 + prev_vsl->set_next(vsl->next()); 4.272 + } 4.273 + 4.274 + vsl->purge(chunk_manager()); 4.275 + dec_virtual_space_total(vsl->reserved()->word_size()); 4.276 + dec_virtual_space_count(); 4.277 + purged_vsl = vsl; 4.278 + delete vsl; 4.279 + } else { 4.280 + prev_vsl = vsl; 4.281 + } 4.282 + } 4.283 +#ifdef ASSERT 4.284 + if (purged_vsl != NULL) { 4.285 + // List should be stable enough to use an iterator here. 4.286 + VirtualSpaceListIterator iter(virtual_space_list()); 4.287 + while (iter.repeat()) { 4.288 + VirtualSpaceNode* vsl = iter.get_next(); 4.289 + assert(vsl != purged_vsl, "Purge of vsl failed"); 4.290 + } 4.291 + } 4.292 +#endif 4.293 +} 4.294 + 4.295 size_t VirtualSpaceList::used_words_sum() { 4.296 size_t allocated_by_vs = 0; 4.297 VirtualSpaceListIterator iter(virtual_space_list()); 4.298 @@ -955,8 +1119,10 @@ 4.299 // Get a chunk from the chunk freelist 4.300 Metachunk* next = chunk_manager()->chunk_freelist_allocate(grow_chunks_by_words); 4.301 4.302 - // Allocate a chunk out of the current virtual space. 4.303 - if (next == NULL) { 4.304 + if (next != NULL) { 4.305 + next->container()->inc_container_count(); 4.306 + } else { 4.307 + // Allocate a chunk out of the current virtual space. 4.308 next = current_virtual_space()->get_chunk_vs(grow_chunks_by_words); 4.309 } 4.310 4.311 @@ -1567,9 +1733,6 @@ 4.312 } 4.313 // Chunk is being removed from the chunks free list. 4.314 dec_free_chunks_total(chunk->capacity_word_size()); 4.315 -#ifdef ASSERT 4.316 - chunk->set_is_free(false); 4.317 -#endif 4.318 } else { 4.319 return NULL; 4.320 } 4.321 @@ -1578,6 +1741,11 @@ 4.322 // Remove it from the links to this freelist 4.323 chunk->set_next(NULL); 4.324 chunk->set_prev(NULL); 4.325 +#ifdef ASSERT 4.326 + // Chunk is no longer on any freelist. Setting to false make container_count_slow() 4.327 + // work. 4.328 + chunk->set_is_free(false); 4.329 +#endif 4.330 slow_locked_verify(); 4.331 return chunk; 4.332 } 4.333 @@ -1887,11 +2055,13 @@ 4.334 assert_lock_strong(SpaceManager::expand_lock()); 4.335 Metachunk* cur = chunks; 4.336 4.337 - // This return chunks one at a time. If a new 4.338 + // This returns chunks one at a time. If a new 4.339 // class List can be created that is a base class 4.340 // of FreeList then something like FreeList::prepend() 4.341 // can be used in place of this loop 4.342 while (cur != NULL) { 4.343 + assert(cur->container() != NULL, "Container should have been set"); 4.344 + cur->container()->dec_container_count(); 4.345 // Capture the next link before it is changed 4.346 // by the call to return_chunk_at_head(); 4.347 Metachunk* next = cur->next(); 4.348 @@ -1917,8 +2087,8 @@ 4.349 locked_print_chunks_in_use_on(gclog_or_tty); 4.350 } 4.351 4.352 - // Mangle freed memory. 4.353 - NOT_PRODUCT(mangle_freed_chunks();) 4.354 + // Do not mangle freed Metachunks. The chunk size inside Metachunks 4.355 + // is during the freeing of a VirtualSpaceNodes. 4.356 4.357 // Have to update before the chunks_in_use lists are emptied 4.358 // below. 4.359 @@ -1978,6 +2148,7 @@ 4.360 " granularity %d", 4.361 humongous_chunks->word_size(), HumongousChunkGranularity)); 4.362 Metachunk* next_humongous_chunks = humongous_chunks->next(); 4.363 + humongous_chunks->container()->dec_container_count(); 4.364 chunk_manager->humongous_dictionary()->return_chunk(humongous_chunks); 4.365 humongous_chunks = next_humongous_chunks; 4.366 } 4.367 @@ -2716,6 +2887,13 @@ 4.368 return Metablock::initialize(result, word_size); 4.369 } 4.370 4.371 +void Metaspace::purge() { 4.372 + MutexLockerEx cl(SpaceManager::expand_lock(), 4.373 + Mutex::_no_safepoint_check_flag); 4.374 + space_list()->purge(); 4.375 + class_space_list()->purge(); 4.376 +} 4.377 + 4.378 void Metaspace::print_on(outputStream* out) const { 4.379 // Print both class virtual space counts and metaspace. 4.380 if (Verbose) { 4.381 @@ -2733,7 +2911,8 @@ 4.382 // aren't deleted presently. When they are, some sort of locking might 4.383 // be needed. Note, locking this can cause inversion problems with the 4.384 // caller in MetaspaceObj::is_metadata() function. 4.385 - return space_list()->contains(ptr) || class_space_list()->contains(ptr); 4.386 + return space_list()->contains(ptr) || 4.387 + class_space_list()->contains(ptr); 4.388 } 4.389 4.390 void Metaspace::verify() {
5.1 --- a/src/share/vm/memory/metaspace.hpp Thu Apr 18 06:50:35 2013 +0200 5.2 +++ b/src/share/vm/memory/metaspace.hpp Fri Mar 01 10:19:29 2013 -0800 5.3 @@ -150,6 +150,9 @@ 5.4 static bool contains(const void *ptr); 5.5 void dump(outputStream* const out) const; 5.6 5.7 + // Free empty virtualspaces 5.8 + static void purge(); 5.9 + 5.10 void print_on(outputStream* st) const; 5.11 // Debugging support 5.12 void verify();