Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "precompiled.hpp"

 #include "gc_interface/collectedHeap.hpp"

 #include "memory/allocation.hpp"

 #include "memory/binaryTreeDictionary.hpp"

 #include "memory/freeList.hpp"

 #include "memory/collectorPolicy.hpp"

 #include "memory/filemap.hpp"

 #include "memory/freeList.hpp"

 #include "memory/gcLocker.hpp"

 #include "memory/metachunk.hpp"

 #include "memory/metaspace.hpp"

 #include "memory/metaspaceGCThresholdUpdater.hpp"

 #include "memory/metaspaceShared.hpp"

 #include "memory/metaspaceTracer.hpp"

 #include "memory/resourceArea.hpp"

 #include "memory/universe.hpp"

 #include "runtime/atomic.inline.hpp"

 #include "runtime/globals.hpp"

 #include "runtime/init.hpp"

 #include "runtime/java.hpp"

 #include "runtime/mutex.hpp"

 #include "runtime/orderAccess.inline.hpp"

 #include "services/memTracker.hpp"

 #include "services/memoryService.hpp"

 #include "utilities/copy.hpp"

 #include "utilities/debug.hpp"

 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC

 typedef BinaryTreeDictionary<Metablock, FreeList<Metablock> > BlockTreeDictionary;

 typedef BinaryTreeDictionary<Metachunk, FreeList<Metachunk> > ChunkTreeDictionary;

 // Set this constant to enable slow integrity checking of the free chunk lists

 const bool metaspace_slow_verify = false;

 size_t const allocation_from_dictionary_limit = 4 * K;

 MetaWord* last_allocated = 0;

 size_t Metaspace::_compressed_class_space_size;

 const MetaspaceTracer* Metaspace::_tracer = NULL;

 // Used in declarations in SpaceManager and ChunkManager

 enum ChunkIndex {

   ZeroIndex = 0,

   SpecializedIndex = ZeroIndex,

   SmallIndex = SpecializedIndex + 1,

   MediumIndex = SmallIndex + 1,

   HumongousIndex = MediumIndex + 1,

   NumberOfFreeLists = 3,

   NumberOfInUseLists = 4

 };

 enum ChunkSizes {    // in words.

   ClassSpecializedChunk = 128,

   SpecializedChunk = 128,

   ClassSmallChunk = 256,

   SmallChunk = 512,

   ClassMediumChunk = 4 * K,

   MediumChunk = 8 * K

 };

 static ChunkIndex next_chunk_index(ChunkIndex i) {

   assert(i < NumberOfInUseLists, "Out of bound");

   return (ChunkIndex) (i+1);

 }

 volatile intptr_t MetaspaceGC::_capacity_until_GC = 0;

 uint MetaspaceGC::_shrink_factor = 0;

 bool MetaspaceGC::_should_concurrent_collect = false;

 typedef class FreeList<Metachunk> ChunkList;

 // Manages the global free lists of chunks.

 class ChunkManager : public CHeapObj<mtInternal> {

   friend class TestVirtualSpaceNodeTest;

   // Free list of chunks of different sizes.

   //   SpecializedChunk

   //   SmallChunk

   //   MediumChunk

   //   HumongousChunk

   ChunkList _free_chunks[NumberOfFreeLists];

   //   HumongousChunk

   ChunkTreeDictionary _humongous_dictionary;

   // ChunkManager in all lists of this type

   size_t _free_chunks_total;

   size_t _free_chunks_count;

   void dec_free_chunks_total(size_t v) {

     assert(_free_chunks_count > 0 &&

              _free_chunks_total > 0,

              "About to go negative");

     Atomic::add_ptr(-1, &_free_chunks_count);

     jlong minus_v = (jlong) - (jlong) v;

     Atomic::add_ptr(minus_v, &_free_chunks_total);

   }

   // Debug support

   size_t sum_free_chunks();

   size_t sum_free_chunks_count();

   void locked_verify_free_chunks_total();

   void slow_locked_verify_free_chunks_total() {

     if (metaspace_slow_verify) {

       locked_verify_free_chunks_total();

     }

   }

   void locked_verify_free_chunks_count();

   void slow_locked_verify_free_chunks_count() {

     if (metaspace_slow_verify) {

       locked_verify_free_chunks_count();

     }

   }

   void verify_free_chunks_count();

  public:

   ChunkManager(size_t specialized_size, size_t small_size, size_t medium_size)

       : _free_chunks_total(0), _free_chunks_count(0) {

     _free_chunks[SpecializedIndex].set_size(specialized_size);

     _free_chunks[SmallIndex].set_size(small_size);

     _free_chunks[MediumIndex].set_size(medium_size);

   }

   // add or delete (return) a chunk to the global freelist.

   Metachunk* chunk_freelist_allocate(size_t word_size);

   // Map a size to a list index assuming that there are lists

   // for special, small, medium, and humongous chunks.

   ChunkIndex list_index(size_t size);

   // Remove the chunk from its freelist.  It is

   // expected to be on one of the _free_chunks[] lists.

   void remove_chunk(Metachunk* chunk);

   // Add the simple linked list of chunks to the freelist of chunks

   // of type index.

   void return_chunks(ChunkIndex index, Metachunk* chunks);

   // Total of the space in the free chunks list

   size_t free_chunks_total_words();

   size_t free_chunks_total_bytes();

   // Number of chunks in the free chunks list

   size_t free_chunks_count();

   void inc_free_chunks_total(size_t v, size_t count = 1) {

     Atomic::add_ptr(count, &_free_chunks_count);

     Atomic::add_ptr(v, &_free_chunks_total);

   }

   ChunkTreeDictionary* humongous_dictionary() {

     return &_humongous_dictionary;

   }

   ChunkList* free_chunks(ChunkIndex index);

   // Returns the list for the given chunk word size.

   ChunkList* find_free_chunks_list(size_t word_size);

   // Remove from a list by size.  Selects list based on size of chunk.

   Metachunk* free_chunks_get(size_t chunk_word_size);

 #define index_bounds_check(index)                                         \

   assert(index == SpecializedIndex ||                                     \

          index == SmallIndex ||                                           \

          index == MediumIndex ||                                          \

          index == HumongousIndex, err_msg("Bad index: %d", (int) index))

   size_t num_free_chunks(ChunkIndex index) const {

     index_bounds_check(index);

     if (index == HumongousIndex) {

       return _humongous_dictionary.total_free_blocks();

     }

     ssize_t count = _free_chunks[index].count();

     return count == -1 ? 0 : (size_t) count;

   }

   size_t size_free_chunks_in_bytes(ChunkIndex index) const {

     index_bounds_check(index);

     size_t word_size = 0;

     if (index == HumongousIndex) {

       word_size = _humongous_dictionary.total_size();

     } else {

       const size_t size_per_chunk_in_words = _free_chunks[index].size();

       word_size = size_per_chunk_in_words * num_free_chunks(index);

     }

     return word_size * BytesPerWord;

   }

   MetaspaceChunkFreeListSummary chunk_free_list_summary() const {

     return MetaspaceChunkFreeListSummary(num_free_chunks(SpecializedIndex),

                                          num_free_chunks(SmallIndex),

                                          num_free_chunks(MediumIndex),

                                          num_free_chunks(HumongousIndex),

                                          size_free_chunks_in_bytes(SpecializedIndex),

                                          size_free_chunks_in_bytes(SmallIndex),

                                          size_free_chunks_in_bytes(MediumIndex),

                                          size_free_chunks_in_bytes(HumongousIndex));

   }

   // Debug support

   void verify();

   void slow_verify() {

     if (metaspace_slow_verify) {

       verify();

     }

   }

   void locked_verify();

   void slow_locked_verify() {

     if (metaspace_slow_verify) {

       locked_verify();

     }

   }

   void verify_free_chunks_total();

   void locked_print_free_chunks(outputStream* st);

   void locked_print_sum_free_chunks(outputStream* st);

   void print_on(outputStream* st) const;

 };

 // Used to manage the free list of Metablocks (a block corresponds

 // to the allocation of a quantum of metadata).

 class BlockFreelist VALUE_OBJ_CLASS_SPEC {

   BlockTreeDictionary* _dictionary;

   // Only allocate and split from freelist if the size of the allocation

   // is at least 1/4th the size of the available block.

   const static int WasteMultiplier = 4;

   // Accessors

   BlockTreeDictionary* dictionary() const { return _dictionary; }

  public:

   BlockFreelist();

   ~BlockFreelist();

   // Get and return a block to the free list

   MetaWord* get_block(size_t word_size);

   void return_block(MetaWord* p, size_t word_size);

   size_t total_size() {

   if (dictionary() == NULL) {

     return 0;

   } else {

     return dictionary()->total_size();

   }

 }

   void print_on(outputStream* st) const;

 };

 // A VirtualSpaceList node.

 class VirtualSpaceNode : public CHeapObj<mtClass> {

   friend class VirtualSpaceList;

   // Link to next VirtualSpaceNode

   VirtualSpaceNode* _next;

   // total in the VirtualSpace

   MemRegion _reserved;

   ReservedSpace _rs;

   VirtualSpace _virtual_space;

   MetaWord* _top;

   // count of chunks contained in this VirtualSpace

   uintx _container_count;

   // Convenience functions to access the _virtual_space

   char* low()  const { return virtual_space()->low(); }

   char* high() const { return virtual_space()->high(); }

   // The first Metachunk will be allocated at the bottom of the

   // VirtualSpace

   Metachunk* first_chunk() { return (Metachunk*) bottom(); }

   // Committed but unused space in the virtual space

   size_t free_words_in_vs() const;

  public:

   VirtualSpaceNode(size_t byte_size);

   VirtualSpaceNode(ReservedSpace rs) : _top(NULL), _next(NULL), _rs(rs), _container_count(0) {}

   ~VirtualSpaceNode();

   // Convenience functions for logical bottom and end

   MetaWord* bottom() const { return (MetaWord*) _virtual_space.low(); }

   MetaWord* end() const { return (MetaWord*) _virtual_space.high(); }

   bool contains(const void* ptr) { return ptr >= low() && ptr < high(); }

   size_t reserved_words() const  { return _virtual_space.reserved_size() / BytesPerWord; }

   size_t committed_words() const { return _virtual_space.actual_committed_size() / BytesPerWord; }

   bool is_pre_committed() const { return _virtual_space.special(); }

   // address of next available space in _virtual_space;

   // Accessors

   VirtualSpaceNode* next() { return _next; }

   void set_next(VirtualSpaceNode* v) { _next = v; }

   void set_reserved(MemRegion const v) { _reserved = v; }

   void set_top(MetaWord* v) { _top = v; }

   // Accessors

   MemRegion* reserved() { return &_reserved; }

   VirtualSpace* virtual_space() const { return (VirtualSpace*) &_virtual_space; }

   // Returns true if "word_size" is available in the VirtualSpace

   bool is_available(size_t word_size) { return word_size <= pointer_delta(end(), _top, sizeof(MetaWord)); }

   MetaWord* top() const { return _top; }

   void inc_top(size_t word_size) { _top += word_size; }

   uintx container_count() { return _container_count; }

   void inc_container_count();

   void dec_container_count();

 #ifdef ASSERT

   uint container_count_slow();

   void verify_container_count();

 #endif

   // used and capacity in this single entry in the list

   size_t used_words_in_vs() const;

   size_t capacity_words_in_vs() const;

   bool initialize();

   // get space from the virtual space

   Metachunk* take_from_committed(size_t chunk_word_size);

   // Allocate a chunk from the virtual space and return it.

   Metachunk* get_chunk_vs(size_t chunk_word_size);

   // Expands/shrinks the committed space in a virtual space.  Delegates

   // to Virtualspace

   bool expand_by(size_t min_words, size_t preferred_words);

   // In preparation for deleting this node, remove all the chunks

   // in the node from any freelist.

   void purge(ChunkManager* chunk_manager);

   // If an allocation doesn't fit in the current node a new node is created.

   // Allocate chunks out of the remaining committed space in this node

   // to avoid wasting that memory.

   // This always adds up because all the chunk sizes are multiples of

   // the smallest chunk size.

   void retire(ChunkManager* chunk_manager);

 #ifdef ASSERT

   // Debug support

   void mangle();

 #endif

   void print_on(outputStream* st) const;

 };

 #define assert_is_ptr_aligned(ptr, alignment) \

   assert(is_ptr_aligned(ptr, alignment),      \

     err_msg(PTR_FORMAT " is not aligned to "  \

       SIZE_FORMAT, ptr, alignment))

 #define assert_is_size_aligned(size, alignment) \

   assert(is_size_aligned(size, alignment),      \

     err_msg(SIZE_FORMAT " is not aligned to "   \

        SIZE_FORMAT, size, alignment))

 // Decide if large pages should be committed when the memory is reserved.

 static bool should_commit_large_pages_when_reserving(size_t bytes) {

   if (UseLargePages && UseLargePagesInMetaspace && !os::can_commit_large_page_memory()) {

     size_t words = bytes / BytesPerWord;

     bool is_class = false; // We never reserve large pages for the class space.

     if (MetaspaceGC::can_expand(words, is_class) &&

         MetaspaceGC::allowed_expansion() >= words) {

       return true;

     }

   }

   return false;

 }

   // byte_size is the size of the associated virtualspace.

 VirtualSpaceNode::VirtualSpaceNode(size_t bytes) : _top(NULL), _next(NULL), _rs(), _container_count(0) {

   assert_is_size_aligned(bytes, Metaspace::reserve_alignment());

 #if INCLUDE_CDS

   // This allocates memory with mmap.  For DumpSharedspaces, try to reserve

   // configurable address, generally at the top of the Java heap so other

   // memory addresses don't conflict.

   if (DumpSharedSpaces) {

     bool large_pages = false; // No large pages when dumping the CDS archive.

     char* shared_base = (char*)align_ptr_up((char*)SharedBaseAddress, Metaspace::reserve_alignment());

     _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages, shared_base, 0);

     if (_rs.is_reserved()) {

       assert(shared_base == 0 || _rs.base() == shared_base, "should match");

     } else {

       // Get a mmap region anywhere if the SharedBaseAddress fails.

       _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages);

     }

     MetaspaceShared::set_shared_rs(&_rs);

   } else

 #endif

   {

     bool large_pages = should_commit_large_pages_when_reserving(bytes);

     _rs = ReservedSpace(bytes, Metaspace::reserve_alignment(), large_pages);

   }

   if (_rs.is_reserved()) {

     assert(_rs.base() != NULL, "Catch if we get a NULL address");

     assert(_rs.size() != 0, "Catch if we get a 0 size");

     assert_is_ptr_aligned(_rs.base(), Metaspace::reserve_alignment());

     assert_is_size_aligned(_rs.size(), Metaspace::reserve_alignment());

     MemTracker::record_virtual_memory_type((address)_rs.base(), mtClass);

   }

 }

 void VirtualSpaceNode::purge(ChunkManager* chunk_manager) {

   Metachunk* chunk = first_chunk();

   Metachunk* invalid_chunk = (Metachunk*) top();

   while (chunk < invalid_chunk ) {

     assert(chunk->is_tagged_free(), "Should be tagged free");

     MetaWord* next = ((MetaWord*)chunk) + chunk->word_size();

     chunk_manager->remove_chunk(chunk);

     assert(chunk->next() == NULL &&

            chunk->prev() == NULL,

            "Was not removed from its list");

     chunk = (Metachunk*) next;

   }

 }

 #ifdef ASSERT

 uint VirtualSpaceNode::container_count_slow() {

   uint count = 0;

   Metachunk* chunk = first_chunk();

   Metachunk* invalid_chunk = (Metachunk*) top();

   while (chunk < invalid_chunk ) {

     MetaWord* next = ((MetaWord*)chunk) + chunk->word_size();

     // Don't count the chunks on the free lists.  Those are

     // still part of the VirtualSpaceNode but not currently

     // counted.

     if (!chunk->is_tagged_free()) {

       count++;

     }

     chunk = (Metachunk*) next;

   }

   return count;

 }

 #endif

 // List of VirtualSpaces for metadata allocation.

 class VirtualSpaceList : public CHeapObj<mtClass> {

   friend class VirtualSpaceNode;

   enum VirtualSpaceSizes {

     VirtualSpaceSize = 256 * K

   };

   // Head of the list

   VirtualSpaceNode* _virtual_space_list;

   // virtual space currently being used for allocations

   VirtualSpaceNode* _current_virtual_space;

   // Is this VirtualSpaceList used for the compressed class space

   bool _is_class;

   // Sum of reserved and committed memory in the virtual spaces

   size_t _reserved_words;

   size_t _committed_words;

   // Number of virtual spaces

   size_t _virtual_space_count;

   ~VirtualSpaceList();

   VirtualSpaceNode* virtual_space_list() const { return _virtual_space_list; }

   void set_virtual_space_list(VirtualSpaceNode* v) {

     _virtual_space_list = v;

   }

   void set_current_virtual_space(VirtualSpaceNode* v) {

     _current_virtual_space = v;

   }

   void link_vs(VirtualSpaceNode* new_entry);

   // Get another virtual space and add it to the list.  This

   // is typically prompted by a failed attempt to allocate a chunk

   // and is typically followed by the allocation of a chunk.

   bool create_new_virtual_space(size_t vs_word_size);

   // Chunk up the unused committed space in the current

   // virtual space and add the chunks to the free list.

   void retire_current_virtual_space();

  public:

   VirtualSpaceList(size_t word_size);

   VirtualSpaceList(ReservedSpace rs);

   size_t free_bytes();

   Metachunk* get_new_chunk(size_t chunk_word_size,

                            size_t suggested_commit_granularity);

   bool expand_node_by(VirtualSpaceNode* node,

                       size_t min_words,

                       size_t preferred_words);

   bool expand_by(size_t min_words,

                  size_t preferred_words);

   VirtualSpaceNode* current_virtual_space() {

     return _current_virtual_space;

   }

   bool is_class() const { return _is_class; }

   bool initialization_succeeded() { return _virtual_space_list != NULL; }

   size_t reserved_words()  { return _reserved_words; }

   size_t reserved_bytes()  { return reserved_words() * BytesPerWord; }

   size_t committed_words() { return _committed_words; }

   size_t committed_bytes() { return committed_words() * BytesPerWord; }

   void inc_reserved_words(size_t v);

   void dec_reserved_words(size_t v);

   void inc_committed_words(size_t v);

   void dec_committed_words(size_t v);

   void inc_virtual_space_count();

   void dec_virtual_space_count();

   bool contains(const void* ptr);

   // Unlink empty VirtualSpaceNodes and free it.

   void purge(ChunkManager* chunk_manager);

   void print_on(outputStream* st) const;

   class VirtualSpaceListIterator : public StackObj {

     VirtualSpaceNode* _virtual_spaces;

    public:

     VirtualSpaceListIterator(VirtualSpaceNode* virtual_spaces) :

       _virtual_spaces(virtual_spaces) {}

     bool repeat() {

       return _virtual_spaces != NULL;

     }

     VirtualSpaceNode* get_next() {

       VirtualSpaceNode* result = _virtual_spaces;

       if (_virtual_spaces != NULL) {

         _virtual_spaces = _virtual_spaces->next();

       }

       return result;

     }

   };

 };

 class Metadebug : AllStatic {

   // Debugging support for Metaspaces

   static int _allocation_fail_alot_count;

  public:

   static void init_allocation_fail_alot_count();

 #ifdef ASSERT

   static bool test_metadata_failure();

 #endif

 };

 int Metadebug::_allocation_fail_alot_count = 0;

 //  SpaceManager - used by Metaspace to handle allocations

 class SpaceManager : public CHeapObj<mtClass> {

   friend class Metaspace;

   friend class Metadebug;

  private:

   // protects allocations

   Mutex* const _lock;

   // Type of metadata allocated.

   Metaspace::MetadataType _mdtype;

   // List of chunks in use by this SpaceManager.  Allocations

   // are done from the current chunk.  The list is used for deallocating

   // chunks when the SpaceManager is freed.

   Metachunk* _chunks_in_use[NumberOfInUseLists];

   Metachunk* _current_chunk;

   // Number of small chunks to allocate to a manager

   // If class space manager, small chunks are unlimited

   static uint const _small_chunk_limit;

   // Sum of all space in allocated chunks

   size_t _allocated_blocks_words;

   // Sum of all allocated chunks

   size_t _allocated_chunks_words;

   size_t _allocated_chunks_count;

   // Free lists of blocks are per SpaceManager since they

   // are assumed to be in chunks in use by the SpaceManager

   // and all chunks in use by a SpaceManager are freed when

   // the class loader using the SpaceManager is collected.

   BlockFreelist _block_freelists;

   // protects virtualspace and chunk expansions

   static const char*  _expand_lock_name;

   static const int    _expand_lock_rank;

   static Mutex* const _expand_lock;

  private:

   // Accessors

   Metachunk* chunks_in_use(ChunkIndex index) const { return _chunks_in_use[index]; }

   void set_chunks_in_use(ChunkIndex index, Metachunk* v) {

     _chunks_in_use[index] = v;

   }

   BlockFreelist* block_freelists() const {

     return (BlockFreelist*) &_block_freelists;

   }

   Metaspace::MetadataType mdtype() { return _mdtype; }

   VirtualSpaceList* vs_list()   const { return Metaspace::get_space_list(_mdtype); }

   ChunkManager* chunk_manager() const { return Metaspace::get_chunk_manager(_mdtype); }

   Metachunk* current_chunk() const { return _current_chunk; }

   void set_current_chunk(Metachunk* v) {

     _current_chunk = v;

   }

   Metachunk* find_current_chunk(size_t word_size);

   // Add chunk to the list of chunks in use

   void add_chunk(Metachunk* v, bool make_current);

   void retire_current_chunk();

   Mutex* lock() const { return _lock; }

   const char* chunk_size_name(ChunkIndex index) const;

  protected:

   void initialize();

  public:

   SpaceManager(Metaspace::MetadataType mdtype,

                Mutex* lock);

   ~SpaceManager();

   enum ChunkMultiples {

     MediumChunkMultiple = 4

   };

   static size_t specialized_chunk_size(bool is_class) { return is_class ? ClassSpecializedChunk : SpecializedChunk; }

   static size_t small_chunk_size(bool is_class)       { return is_class ? ClassSmallChunk : SmallChunk; }

   static size_t medium_chunk_size(bool is_class)      { return is_class ? ClassMediumChunk : MediumChunk; }

   static size_t smallest_chunk_size(bool is_class)    { return specialized_chunk_size(is_class); }

   // Accessors

   bool is_class() const { return _mdtype == Metaspace::ClassType; }

   size_t specialized_chunk_size() const { return specialized_chunk_size(is_class()); }

   size_t small_chunk_size()       const { return small_chunk_size(is_class()); }

   size_t medium_chunk_size()      const { return medium_chunk_size(is_class()); }

   size_t smallest_chunk_size()    const { return smallest_chunk_size(is_class()); }

   size_t medium_chunk_bunch()     const { return medium_chunk_size() * MediumChunkMultiple; }

   size_t allocated_blocks_words() const { return _allocated_blocks_words; }

   size_t allocated_blocks_bytes() const { return _allocated_blocks_words * BytesPerWord; }

   size_t allocated_chunks_words() const { return _allocated_chunks_words; }

   size_t allocated_chunks_bytes() const { return _allocated_chunks_words * BytesPerWord; }

   size_t allocated_chunks_count() const { return _allocated_chunks_count; }

   bool is_humongous(size_t word_size) { return word_size > medium_chunk_size(); }

   static Mutex* expand_lock() { return _expand_lock; }

   // Increment the per Metaspace and global running sums for Metachunks

   // by the given size.  This is used when a Metachunk to added to

   // the in-use list.

   void inc_size_metrics(size_t words);

   // Increment the per Metaspace and global running sums Metablocks by the given

   // size.  This is used when a Metablock is allocated.

   void inc_used_metrics(size_t words);

   // Delete the portion of the running sums for this SpaceManager. That is,

   // the globals running sums for the Metachunks and Metablocks are

   // decremented for all the Metachunks in-use by this SpaceManager.

   void dec_total_from_size_metrics();

   // Adjust the initial chunk size to match one of the fixed chunk list sizes,

   // or return the unadjusted size if the requested size is humongous.

   static size_t adjust_initial_chunk_size(size_t requested, bool is_class_space);

   size_t adjust_initial_chunk_size(size_t requested) const;

   // Get the initial chunks size for this metaspace type.

   size_t get_initial_chunk_size(Metaspace::MetaspaceType type) const;

   size_t sum_capacity_in_chunks_in_use() const;

   size_t sum_used_in_chunks_in_use() const;

   size_t sum_free_in_chunks_in_use() const;

   size_t sum_waste_in_chunks_in_use() const;

   size_t sum_waste_in_chunks_in_use(ChunkIndex index ) const;

   size_t sum_count_in_chunks_in_use();

   size_t sum_count_in_chunks_in_use(ChunkIndex i);

   Metachunk* get_new_chunk(size_t chunk_word_size);

   // Block allocation and deallocation.

   // Allocates a block from the current chunk

   MetaWord* allocate(size_t word_size);

   // Helper for allocations

   MetaWord* allocate_work(size_t word_size);

   // Returns a block to the per manager freelist

   void deallocate(MetaWord* p, size_t word_size);

   // Based on the allocation size and a minimum chunk size,

   // returned chunk size (for expanding space for chunk allocation).

   size_t calc_chunk_size(size_t allocation_word_size);

   // Called when an allocation from the current chunk fails.

   // Gets a new chunk (may require getting a new virtual space),

   // and allocates from that chunk.

   MetaWord* grow_and_allocate(size_t word_size);

   // Notify memory usage to MemoryService.

   void track_metaspace_memory_usage();

   // debugging support.

   void dump(outputStream* const out) const;

   void print_on(outputStream* st) const;

   void locked_print_chunks_in_use_on(outputStream* st) const;

   void verify();

   void verify_chunk_size(Metachunk* chunk);

   NOT_PRODUCT(void mangle_freed_chunks();)

 #ifdef ASSERT

   void verify_allocated_blocks_words();

 #endif

   size_t get_raw_word_size(size_t word_size) {

     size_t byte_size = word_size * BytesPerWord;

     size_t raw_bytes_size = MAX2(byte_size, sizeof(Metablock));

     raw_bytes_size = align_size_up(raw_bytes_size, Metachunk::object_alignment());

     size_t raw_word_size = raw_bytes_size / BytesPerWord;

     assert(raw_word_size * BytesPerWord == raw_bytes_size, "Size problem");

     return raw_word_size;

   }

 };

 uint const SpaceManager::_small_chunk_limit = 4;

 const char* SpaceManager::_expand_lock_name =

   "SpaceManager chunk allocation lock";

 const int SpaceManager::_expand_lock_rank = Monitor::leaf - 1;

 Mutex* const SpaceManager::_expand_lock =

   new Mutex(SpaceManager::_expand_lock_rank,

             SpaceManager::_expand_lock_name,

             Mutex::_allow_vm_block_flag);

 void VirtualSpaceNode::inc_container_count() {

   assert_lock_strong(SpaceManager::expand_lock());

   _container_count++;

   assert(_container_count == container_count_slow(),

          err_msg("Inconsistency in countainer_count _container_count " SIZE_FORMAT

                  " container_count_slow() " SIZE_FORMAT,

                  _container_count, container_count_slow()));

 }

 void VirtualSpaceNode::dec_container_count() {

   assert_lock_strong(SpaceManager::expand_lock());

   _container_count--;

 }

 #ifdef ASSERT

 void VirtualSpaceNode::verify_container_count() {

   assert(_container_count == container_count_slow(),

     err_msg("Inconsistency in countainer_count _container_count " SIZE_FORMAT

             " container_count_slow() " SIZE_FORMAT, _container_count, container_count_slow()));

 }

 #endif

 // BlockFreelist methods

 BlockFreelist::BlockFreelist() : _dictionary(NULL) {}

 BlockFreelist::~BlockFreelist() {

   if (_dictionary != NULL) {

     if (Verbose && TraceMetadataChunkAllocation) {

       _dictionary->print_free_lists(gclog_or_tty);

     }

     delete _dictionary;

   }

 }

 void BlockFreelist::return_block(MetaWord* p, size_t word_size) {

   Metablock* free_chunk = ::new (p) Metablock(word_size);

   if (dictionary() == NULL) {

    _dictionary = new BlockTreeDictionary();

   }

   dictionary()->return_chunk(free_chunk);

 }

 MetaWord* BlockFreelist::get_block(size_t word_size) {

   if (dictionary() == NULL) {

     return NULL;

   }

   if (word_size < TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {

     // Dark matter.  Too small for dictionary.

     return NULL;

   }

   Metablock* free_block =

     dictionary()->get_chunk(word_size, FreeBlockDictionary<Metablock>::atLeast);

   if (free_block == NULL) {

     return NULL;

   }

   const size_t block_size = free_block->size();

   if (block_size > WasteMultiplier * word_size) {

     return_block((MetaWord*)free_block, block_size);

     return NULL;

   }

   MetaWord* new_block = (MetaWord*)free_block;

   assert(block_size >= word_size, "Incorrect size of block from freelist");

   const size_t unused = block_size - word_size;

   if (unused >= TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {

     return_block(new_block + word_size, unused);

   }

   return new_block;

 }

 void BlockFreelist::print_on(outputStream* st) const {

   if (dictionary() == NULL) {

     return;

   }

   dictionary()->print_free_lists(st);

 }

 // VirtualSpaceNode methods

 VirtualSpaceNode::~VirtualSpaceNode() {

   _rs.release();

 #ifdef ASSERT

   size_t word_size = sizeof(*this) / BytesPerWord;

   Copy::fill_to_words((HeapWord*) this, word_size, 0xf1f1f1f1);

 #endif

 }

 size_t VirtualSpaceNode::used_words_in_vs() const {

   return pointer_delta(top(), bottom(), sizeof(MetaWord));

 }

 // Space committed in the VirtualSpace

 size_t VirtualSpaceNode::capacity_words_in_vs() const {

   return pointer_delta(end(), bottom(), sizeof(MetaWord));

 }

 size_t VirtualSpaceNode::free_words_in_vs() const {

   return pointer_delta(end(), top(), sizeof(MetaWord));

 }

 // Allocates the chunk from the virtual space only.

 // This interface is also used internally for debugging.  Not all

 // chunks removed here are necessarily used for allocation.

 Metachunk* VirtualSpaceNode::take_from_committed(size_t chunk_word_size) {

   // Bottom of the new chunk

   MetaWord* chunk_limit = top();

   assert(chunk_limit != NULL, "Not safe to call this method");

   // The virtual spaces are always expanded by the

   // commit granularity to enforce the following condition.

   // Without this the is_available check will not work correctly.

   assert(_virtual_space.committed_size() == _virtual_space.actual_committed_size(),

       "The committed memory doesn't match the expanded memory.");

   if (!is_available(chunk_word_size)) {

     if (TraceMetadataChunkAllocation) {

       gclog_or_tty->print("VirtualSpaceNode::take_from_committed() not available %d words ", chunk_word_size);

       // Dump some information about the virtual space that is nearly full

       print_on(gclog_or_tty);

     }

     return NULL;

   }

   // Take the space  (bump top on the current virtual space).

   inc_top(chunk_word_size);

   // Initialize the chunk

   Metachunk* result = ::new (chunk_limit) Metachunk(chunk_word_size, this);

   return result;

 }

 // Expand the virtual space (commit more of the reserved space)

 bool VirtualSpaceNode::expand_by(size_t min_words, size_t preferred_words) {

   size_t min_bytes = min_words * BytesPerWord;

   size_t preferred_bytes = preferred_words * BytesPerWord;

   size_t uncommitted = virtual_space()->reserved_size() - virtual_space()->actual_committed_size();

   if (uncommitted < min_bytes) {

     return false;

   }

   size_t commit = MIN2(preferred_bytes, uncommitted);

   bool result = virtual_space()->expand_by(commit, false);

   assert(result, "Failed to commit memory");

   return result;

 }

 Metachunk* VirtualSpaceNode::get_chunk_vs(size_t chunk_word_size) {

   assert_lock_strong(SpaceManager::expand_lock());

   Metachunk* result = take_from_committed(chunk_word_size);

   if (result != NULL) {

     inc_container_count();

   }

   return result;

 }

 bool VirtualSpaceNode::initialize() {

   if (!_rs.is_reserved()) {

     return false;

   }

   // These are necessary restriction to make sure that the virtual space always

   // grows in steps of Metaspace::commit_alignment(). If both base and size are

   // aligned only the middle alignment of the VirtualSpace is used.

   assert_is_ptr_aligned(_rs.base(), Metaspace::commit_alignment());

   assert_is_size_aligned(_rs.size(), Metaspace::commit_alignment());

   // ReservedSpaces marked as special will have the entire memory

   // pre-committed. Setting a committed size will make sure that

   // committed_size and actual_committed_size agrees.

   size_t pre_committed_size = _rs.special() ? _rs.size() : 0;

   bool result = virtual_space()->initialize_with_granularity(_rs, pre_committed_size,

                                             Metaspace::commit_alignment());

   if (result) {

     assert(virtual_space()->committed_size() == virtual_space()->actual_committed_size(),

         "Checking that the pre-committed memory was registered by the VirtualSpace");

     set_top((MetaWord*)virtual_space()->low());

     set_reserved(MemRegion((HeapWord*)_rs.base(),

                  (HeapWord*)(_rs.base() + _rs.size())));

     assert(reserved()->start() == (HeapWord*) _rs.base(),

       err_msg("Reserved start was not set properly " PTR_FORMAT

         " != " PTR_FORMAT, reserved()->start(), _rs.base()));

     assert(reserved()->word_size() == _rs.size() / BytesPerWord,

       err_msg("Reserved size was not set properly " SIZE_FORMAT

         " != " SIZE_FORMAT, reserved()->word_size(),

         _rs.size() / BytesPerWord));

   }

   return result;

 }

 void VirtualSpaceNode::print_on(outputStream* st) const {

   size_t used = used_words_in_vs();

   size_t capacity = capacity_words_in_vs();

   VirtualSpace* vs = virtual_space();

   st->print_cr("   space @ " PTR_FORMAT " " SIZE_FORMAT "K, %3d%% used "

            "[" PTR_FORMAT ", " PTR_FORMAT ", "

            PTR_FORMAT ", " PTR_FORMAT ")",

            vs, capacity / K,

            capacity == 0 ? 0 : used * 100 / capacity,

            bottom(), top(), end(),

            vs->high_boundary());

 }

 #ifdef ASSERT

 void VirtualSpaceNode::mangle() {

   size_t word_size = capacity_words_in_vs();

   Copy::fill_to_words((HeapWord*) low(), word_size, 0xf1f1f1f1);

 }

 #endif // ASSERT

 // VirtualSpaceList methods

 // Space allocated from the VirtualSpace

 VirtualSpaceList::~VirtualSpaceList() {

   VirtualSpaceListIterator iter(virtual_space_list());

   while (iter.repeat()) {

     VirtualSpaceNode* vsl = iter.get_next();

     delete vsl;

   }

 }

 void VirtualSpaceList::inc_reserved_words(size_t v) {

   assert_lock_strong(SpaceManager::expand_lock());

   _reserved_words = _reserved_words + v;

 }

 void VirtualSpaceList::dec_reserved_words(size_t v) {

   assert_lock_strong(SpaceManager::expand_lock());

   _reserved_words = _reserved_words - v;

 }

 #define assert_committed_below_limit()                             \

   assert(MetaspaceAux::committed_bytes() <= MaxMetaspaceSize,      \

       err_msg("Too much committed memory. Committed: " SIZE_FORMAT \

               " limit (MaxMetaspaceSize): " SIZE_FORMAT,           \

           MetaspaceAux::committed_bytes(), MaxMetaspaceSize));

 void VirtualSpaceList::inc_committed_words(size_t v) {

   assert_lock_strong(SpaceManager::expand_lock());

   _committed_words = _committed_words + v;

   assert_committed_below_limit();

 }

 void VirtualSpaceList::dec_committed_words(size_t v) {

   assert_lock_strong(SpaceManager::expand_lock());

   _committed_words = _committed_words - v;

   assert_committed_below_limit();

 }

 void VirtualSpaceList::inc_virtual_space_count() {

   assert_lock_strong(SpaceManager::expand_lock());

   _virtual_space_count++;

 }

 void VirtualSpaceList::dec_virtual_space_count() {

   assert_lock_strong(SpaceManager::expand_lock());

   _virtual_space_count--;

 }

 void ChunkManager::remove_chunk(Metachunk* chunk) {

   size_t word_size = chunk->word_size();

   ChunkIndex index = list_index(word_size);

   if (index != HumongousIndex) {

     free_chunks(index)->remove_chunk(chunk);

   } else {

     humongous_dictionary()->remove_chunk(chunk);

   }

   // Chunk is being removed from the chunks free list.

   dec_free_chunks_total(chunk->word_size());

 }

 // Walk the list of VirtualSpaceNodes and delete

 // nodes with a 0 container_count.  Remove Metachunks in

 // the node from their respective freelists.

 void VirtualSpaceList::purge(ChunkManager* chunk_manager) {

   assert(SafepointSynchronize::is_at_safepoint(), "must be called at safepoint for contains to work");

   assert_lock_strong(SpaceManager::expand_lock());

   // Don't use a VirtualSpaceListIterator because this

   // list is being changed and a straightforward use of an iterator is not safe.

   VirtualSpaceNode* purged_vsl = NULL;

   VirtualSpaceNode* prev_vsl = virtual_space_list();

   VirtualSpaceNode* next_vsl = prev_vsl;

   while (next_vsl != NULL) {

     VirtualSpaceNode* vsl = next_vsl;

     next_vsl = vsl->next();

     // Don't free the current virtual space since it will likely

     // be needed soon.

     if (vsl->container_count() == 0 && vsl != current_virtual_space()) {

       // Unlink it from the list

       if (prev_vsl == vsl) {

         // This is the case of the current node being the first node.

         assert(vsl == virtual_space_list(), "Expected to be the first node");

         set_virtual_space_list(vsl->next());

       } else {

         prev_vsl->set_next(vsl->next());

       }

       vsl->purge(chunk_manager);

       dec_reserved_words(vsl->reserved_words());

       dec_committed_words(vsl->committed_words());

       dec_virtual_space_count();

       purged_vsl = vsl;

       delete vsl;

     } else {

       prev_vsl = vsl;

     }

   }

 #ifdef ASSERT

   if (purged_vsl != NULL) {

     // List should be stable enough to use an iterator here.

     VirtualSpaceListIterator iter(virtual_space_list());

     while (iter.repeat()) {

       VirtualSpaceNode* vsl = iter.get_next();

       assert(vsl != purged_vsl, "Purge of vsl failed");

     }

   }

 #endif

 }

 // This function looks at the mmap regions in the metaspace without locking.

 // The chunks are added with store ordering and not deleted except for at

 // unloading time during a safepoint.

 bool VirtualSpaceList::contains(const void* ptr) {

   // List should be stable enough to use an iterator here because removing virtual

   // space nodes is only allowed at a safepoint.

   VirtualSpaceListIterator iter(virtual_space_list());

   while (iter.repeat()) {

     VirtualSpaceNode* vsn = iter.get_next();

     if (vsn->contains(ptr)) {

       return true;

     }

   }

   return false;

 }

 void VirtualSpaceList::retire_current_virtual_space() {

   assert_lock_strong(SpaceManager::expand_lock());

   VirtualSpaceNode* vsn = current_virtual_space();

   ChunkManager* cm = is_class() ? Metaspace::chunk_manager_class() :

                                   Metaspace::chunk_manager_metadata();

   vsn->retire(cm);

 }

 void VirtualSpaceNode::retire(ChunkManager* chunk_manager) {

   for (int i = (int)MediumIndex; i >= (int)ZeroIndex; --i) {

     ChunkIndex index = (ChunkIndex)i;

     size_t chunk_size = chunk_manager->free_chunks(index)->size();

     while (free_words_in_vs() >= chunk_size) {

       DEBUG_ONLY(verify_container_count();)

       Metachunk* chunk = get_chunk_vs(chunk_size);

       assert(chunk != NULL, "allocation should have been successful");

       chunk_manager->return_chunks(index, chunk);

       chunk_manager->inc_free_chunks_total(chunk_size);

       DEBUG_ONLY(verify_container_count();)

     }

   }

   assert(free_words_in_vs() == 0, "should be empty now");

 }

 VirtualSpaceList::VirtualSpaceList(size_t word_size) :

                                    _is_class(false),

                                    _virtual_space_list(NULL),

                                    _current_virtual_space(NULL),

                                    _reserved_words(0),

                                    _committed_words(0),

                                    _virtual_space_count(0) {

   MutexLockerEx cl(SpaceManager::expand_lock(),

                    Mutex::_no_safepoint_check_flag);

   create_new_virtual_space(word_size);

 }

 VirtualSpaceList::VirtualSpaceList(ReservedSpace rs) :

                                    _is_class(true),

                                    _virtual_space_list(NULL),

                                    _current_virtual_space(NULL),

                                    _reserved_words(0),

                                    _committed_words(0),

                                    _virtual_space_count(0) {

   MutexLockerEx cl(SpaceManager::expand_lock(),

                    Mutex::_no_safepoint_check_flag);

   VirtualSpaceNode* class_entry = new VirtualSpaceNode(rs);

   bool succeeded = class_entry->initialize();

   if (succeeded) {

     link_vs(class_entry);

   }

 }

 size_t VirtualSpaceList::free_bytes() {

   return current_virtual_space()->free_words_in_vs() * BytesPerWord;

 }

 // Allocate another meta virtual space and add it to the list.

 bool VirtualSpaceList::create_new_virtual_space(size_t vs_word_size) {

   assert_lock_strong(SpaceManager::expand_lock());

   if (is_class()) {

     assert(false, "We currently don't support more than one VirtualSpace for"

                   " the compressed class space. The initialization of the"

                   " CCS uses another code path and should not hit this path.");

     return false;

   }

   if (vs_word_size == 0) {

     assert(false, "vs_word_size should always be at least _reserve_alignment large.");

     return false;

   }

   // Reserve the space

   size_t vs_byte_size = vs_word_size * BytesPerWord;

   assert_is_size_aligned(vs_byte_size, Metaspace::reserve_alignment());

   // Allocate the meta virtual space and initialize it.

   VirtualSpaceNode* new_entry = new VirtualSpaceNode(vs_byte_size);

   if (!new_entry->initialize()) {

     delete new_entry;

     return false;

   } else {

     assert(new_entry->reserved_words() == vs_word_size,

         "Reserved memory size differs from requested memory size");

     // ensure lock-free iteration sees fully initialized node

     OrderAccess::storestore();

     link_vs(new_entry);

     return true;

   }

 }

 void VirtualSpaceList::link_vs(VirtualSpaceNode* new_entry) {

   if (virtual_space_list() == NULL) {

       set_virtual_space_list(new_entry);

   } else {

     current_virtual_space()->set_next(new_entry);

   }

   set_current_virtual_space(new_entry);

   inc_reserved_words(new_entry->reserved_words());

   inc_committed_words(new_entry->committed_words());

   inc_virtual_space_count();

 #ifdef ASSERT

   new_entry->mangle();

 #endif

   if (TraceMetavirtualspaceAllocation && Verbose) {

     VirtualSpaceNode* vsl = current_virtual_space();

     vsl->print_on(gclog_or_tty);

   }

 }

 bool VirtualSpaceList::expand_node_by(VirtualSpaceNode* node,

                                       size_t min_words,

                                       size_t preferred_words) {

   size_t before = node->committed_words();

   bool result = node->expand_by(min_words, preferred_words);

   size_t after = node->committed_words();

   // after and before can be the same if the memory was pre-committed.

   assert(after >= before, "Inconsistency");

   inc_committed_words(after - before);

   return result;

 }

 bool VirtualSpaceList::expand_by(size_t min_words, size_t preferred_words) {

   assert_is_size_aligned(min_words,       Metaspace::commit_alignment_words());

   assert_is_size_aligned(preferred_words, Metaspace::commit_alignment_words());

   assert(min_words <= preferred_words, "Invalid arguments");

   if (!MetaspaceGC::can_expand(min_words, this->is_class())) {

     return  false;

   }

   size_t allowed_expansion_words = MetaspaceGC::allowed_expansion();

   if (allowed_expansion_words < min_words) {

     return false;

   }

   size_t max_expansion_words = MIN2(preferred_words, allowed_expansion_words);

   // Commit more memory from the the current virtual space.

   bool vs_expanded = expand_node_by(current_virtual_space(),

                                     min_words,

                                     max_expansion_words);

   if (vs_expanded) {

     return true;

   }

   retire_current_virtual_space();

   // Get another virtual space.

   size_t grow_vs_words = MAX2((size_t)VirtualSpaceSize, preferred_words);

   grow_vs_words = align_size_up(grow_vs_words, Metaspace::reserve_alignment_words());

   if (create_new_virtual_space(grow_vs_words)) {

     if (current_virtual_space()->is_pre_committed()) {

       // The memory was pre-committed, so we are done here.

       assert(min_words <= current_virtual_space()->committed_words(),

           "The new VirtualSpace was pre-committed, so it"

           "should be large enough to fit the alloc request.");

       return true;

     }

     return expand_node_by(current_virtual_space(),

                           min_words,

                           max_expansion_words);

   }

   return false;

 }

 Metachunk* VirtualSpaceList::get_new_chunk(size_t chunk_word_size, size_t suggested_commit_granularity) {

   // Allocate a chunk out of the current virtual space.

   Metachunk* next = current_virtual_space()->get_chunk_vs(chunk_word_size);

   if (next != NULL) {

     return next;

   }

   // The expand amount is currently only determined by the requested sizes

   // and not how much committed memory is left in the current virtual space.

   size_t min_word_size       = align_size_up(chunk_word_size,              Metaspace::commit_alignment_words());

   size_t preferred_word_size = align_size_up(suggested_commit_granularity, Metaspace::commit_alignment_words());

   if (min_word_size >= preferred_word_size) {

     // Can happen when humongous chunks are allocated.

     preferred_word_size = min_word_size;

   }

   bool expanded = expand_by(min_word_size, preferred_word_size);

   if (expanded) {

     next = current_virtual_space()->get_chunk_vs(chunk_word_size);

     assert(next != NULL, "The allocation was expected to succeed after the expansion");

   }

    return next;

 }

 void VirtualSpaceList::print_on(outputStream* st) const {

   if (TraceMetadataChunkAllocation && Verbose) {

     VirtualSpaceListIterator iter(virtual_space_list());

     while (iter.repeat()) {

       VirtualSpaceNode* node = iter.get_next();

       node->print_on(st);

     }

   }

 }

 // MetaspaceGC methods

 // VM_CollectForMetadataAllocation is the vm operation used to GC.

 // Within the VM operation after the GC the attempt to allocate the metadata

 // should succeed.  If the GC did not free enough space for the metaspace

 // allocation, the HWM is increased so that another virtualspace will be

 // allocated for the metadata.  With perm gen the increase in the perm

 // gen had bounds, MinMetaspaceExpansion and MaxMetaspaceExpansion.  The

 // metaspace policy uses those as the small and large steps for the HWM.

 //

 // After the GC the compute_new_size() for MetaspaceGC is called to

 // resize the capacity of the metaspaces.  The current implementation

 // is based on the flags MinMetaspaceFreeRatio and MaxMetaspaceFreeRatio used

 // to resize the Java heap by some GC's.  New flags can be implemented

 // if really needed.  MinMetaspaceFreeRatio is used to calculate how much

 // free space is desirable in the metaspace capacity to decide how much

 // to increase the HWM.  MaxMetaspaceFreeRatio is used to decide how much

 // free space is desirable in the metaspace capacity before decreasing

 // the HWM.

 // Calculate the amount to increase the high water mark (HWM).

 // Increase by a minimum amount (MinMetaspaceExpansion) so that

 // another expansion is not requested too soon.  If that is not

 // enough to satisfy the allocation, increase by MaxMetaspaceExpansion.

 // If that is still not enough, expand by the size of the allocation

 // plus some.

 size_t MetaspaceGC::delta_capacity_until_GC(size_t bytes) {

   size_t min_delta = MinMetaspaceExpansion;

   size_t max_delta = MaxMetaspaceExpansion;

   size_t delta = align_size_up(bytes, Metaspace::commit_alignment());

   if (delta <= min_delta) {

     delta = min_delta;

   } else if (delta <= max_delta) {

     // Don't want to hit the high water mark on the next

     // allocation so make the delta greater than just enough

     // for this allocation.

     delta = max_delta;

   } else {

     // This allocation is large but the next ones are probably not

     // so increase by the minimum.

     delta = delta + min_delta;

   }

   assert_is_size_aligned(delta, Metaspace::commit_alignment());

   return delta;

 }

 size_t MetaspaceGC::capacity_until_GC() {

   size_t value = (size_t)OrderAccess::load_ptr_acquire(&_capacity_until_GC);

   assert(value >= MetaspaceSize, "Not initialied properly?");

   return value;

 }

 // Try to increase the _capacity_until_GC limit counter by v bytes.

 // Returns true if it succeeded. It may fail if either another thread

 // concurrently increased the limit or the new limit would be larger

 // than MaxMetaspaceSize.

 // On success, optionally returns new and old metaspace capacity in

 // new_cap_until_GC and old_cap_until_GC respectively.

 // On error, optionally sets can_retry to indicate whether if there is

 // actually enough space remaining to satisfy the request.

 bool MetaspaceGC::inc_capacity_until_GC(size_t v, size_t* new_cap_until_GC, size_t* old_cap_until_GC, bool* can_retry) {

   assert_is_size_aligned(v, Metaspace::commit_alignment());

   size_t capacity_until_GC = (size_t) _capacity_until_GC;

   size_t new_value = capacity_until_GC + v;

   if (new_value < capacity_until_GC) {

     // The addition wrapped around, set new_value to aligned max value.

     new_value = align_size_down(max_uintx, Metaspace::commit_alignment());

   }

   if (new_value > MaxMetaspaceSize) {

     if (can_retry != NULL) {

       *can_retry = false;

     }

     return false;

   }

   if (can_retry != NULL) {

     *can_retry = true;

   }

   intptr_t expected = (intptr_t) capacity_until_GC;

   intptr_t actual = Atomic::cmpxchg_ptr((intptr_t) new_value, &_capacity_until_GC, expected);

   if (expected != actual) {

     return false;

   }

   if (new_cap_until_GC != NULL) {

     *new_cap_until_GC = new_value;

   }

   if (old_cap_until_GC != NULL) {

     *old_cap_until_GC = capacity_until_GC;

   }

   return true;

 }

 size_t MetaspaceGC::dec_capacity_until_GC(size_t v) {

   assert_is_size_aligned(v, Metaspace::commit_alignment());

   return (size_t)Atomic::add_ptr(-(intptr_t)v, &_capacity_until_GC);

 }

 void MetaspaceGC::initialize() {

   // Set the high-water mark to MaxMetapaceSize during VM initializaton since

   // we can't do a GC during initialization.

   _capacity_until_GC = MaxMetaspaceSize;

 }

 void MetaspaceGC::post_initialize() {

   // Reset the high-water mark once the VM initialization is done.

   _capacity_until_GC = MAX2(MetaspaceAux::committed_bytes(), MetaspaceSize);

 }

 bool MetaspaceGC::can_expand(size_t word_size, bool is_class) {

   // Check if the compressed class space is full.

   if (is_class && Metaspace::using_class_space()) {

     size_t class_committed = MetaspaceAux::committed_bytes(Metaspace::ClassType);

     if (class_committed + word_size * BytesPerWord > CompressedClassSpaceSize) {

       return false;

     }

   }

   // Check if the user has imposed a limit on the metaspace memory.

   size_t committed_bytes = MetaspaceAux::committed_bytes();

   if (committed_bytes + word_size * BytesPerWord > MaxMetaspaceSize) {

     return false;

   }

   return true;

 }

 size_t MetaspaceGC::allowed_expansion() {

   size_t committed_bytes = MetaspaceAux::committed_bytes();

   size_t capacity_until_gc = capacity_until_GC();

   assert(capacity_until_gc >= committed_bytes,

         err_msg("capacity_until_gc: " SIZE_FORMAT " < committed_bytes: " SIZE_FORMAT,

                 capacity_until_gc, committed_bytes));

   size_t left_until_max  = MaxMetaspaceSize - committed_bytes;

   size_t left_until_GC = capacity_until_gc - committed_bytes;

   size_t left_to_commit = MIN2(left_until_GC, left_until_max);

   return left_to_commit / BytesPerWord;

 }

 void MetaspaceGC::compute_new_size() {

   assert(_shrink_factor <= 100, "invalid shrink factor");

   uint current_shrink_factor = _shrink_factor;

   _shrink_factor = 0;

   // Using committed_bytes() for used_after_gc is an overestimation, since the

   // chunk free lists are included in committed_bytes() and the memory in an

   // un-fragmented chunk free list is available for future allocations.

   // However, if the chunk free lists becomes fragmented, then the memory may

   // not be available for future allocations and the memory is therefore "in use".

   // Including the chunk free lists in the definition of "in use" is therefore

   // necessary. Not including the chunk free lists can cause capacity_until_GC to

   // shrink below committed_bytes() and this has caused serious bugs in the past.

   const size_t used_after_gc = MetaspaceAux::committed_bytes();

   const size_t capacity_until_GC = MetaspaceGC::capacity_until_GC();

   const double minimum_free_percentage = MinMetaspaceFreeRatio / 100.0;

   const double maximum_used_percentage = 1.0 - minimum_free_percentage;

   const double min_tmp = used_after_gc / maximum_used_percentage;

   size_t minimum_desired_capacity =

     (size_t)MIN2(min_tmp, double(MaxMetaspaceSize));

   // Don't shrink less than the initial generation size

   minimum_desired_capacity = MAX2(minimum_desired_capacity,

                                   MetaspaceSize);

   if (PrintGCDetails && Verbose) {

     gclog_or_tty->print_cr("\nMetaspaceGC::compute_new_size: ");

     gclog_or_tty->print_cr("  "

                   "  minimum_free_percentage: %6.2f"

                   "  maximum_used_percentage: %6.2f",

                   minimum_free_percentage,

                   maximum_used_percentage);

     gclog_or_tty->print_cr("  "

                   "   used_after_gc       : %6.1fKB",

                   used_after_gc / (double) K);

   }

   size_t shrink_bytes = 0;

   if (capacity_until_GC < minimum_desired_capacity) {

     // If we have less capacity below the metaspace HWM, then

     // increment the HWM.

     size_t expand_bytes = minimum_desired_capacity - capacity_until_GC;

     expand_bytes = align_size_up(expand_bytes, Metaspace::commit_alignment());

     // Don't expand unless it's significant

     if (expand_bytes >= MinMetaspaceExpansion) {

       size_t new_capacity_until_GC = 0;

       bool succeeded = MetaspaceGC::inc_capacity_until_GC(expand_bytes, &new_capacity_until_GC);

       assert(succeeded, "Should always succesfully increment HWM when at safepoint");

       Metaspace::tracer()->report_gc_threshold(capacity_until_GC,

                                                new_capacity_until_GC,

                                                MetaspaceGCThresholdUpdater::ComputeNewSize);

       if (PrintGCDetails && Verbose) {

         gclog_or_tty->print_cr("    expanding:"

                       "  minimum_desired_capacity: %6.1fKB"

                       "  expand_bytes: %6.1fKB"

                       "  MinMetaspaceExpansion: %6.1fKB"

                       "  new metaspace HWM:  %6.1fKB",

                       minimum_desired_capacity / (double) K,

                       expand_bytes / (double) K,

                       MinMetaspaceExpansion / (double) K,

                       new_capacity_until_GC / (double) K);

       }

     }

     return;

   }

   // No expansion, now see if we want to shrink

   // We would never want to shrink more than this

   size_t max_shrink_bytes = capacity_until_GC - minimum_desired_capacity;

   assert(max_shrink_bytes >= 0, err_msg("max_shrink_bytes " SIZE_FORMAT,

     max_shrink_bytes));

   // Should shrinking be considered?

   if (MaxMetaspaceFreeRatio < 100) {

     const double maximum_free_percentage = MaxMetaspaceFreeRatio / 100.0;

     const double minimum_used_percentage = 1.0 - maximum_free_percentage;

     const double max_tmp = used_after_gc / minimum_used_percentage;

     size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(MaxMetaspaceSize));

     maximum_desired_capacity = MAX2(maximum_desired_capacity,

                                     MetaspaceSize);

     if (PrintGCDetails && Verbose) {

       gclog_or_tty->print_cr("  "

                              "  maximum_free_percentage: %6.2f"

                              "  minimum_used_percentage: %6.2f",

                              maximum_free_percentage,

                              minimum_used_percentage);

       gclog_or_tty->print_cr("  "

                              "  minimum_desired_capacity: %6.1fKB"

                              "  maximum_desired_capacity: %6.1fKB",

                              minimum_desired_capacity / (double) K,

                              maximum_desired_capacity / (double) K);

     }

     assert(minimum_desired_capacity <= maximum_desired_capacity,

            "sanity check");

     if (capacity_until_GC > maximum_desired_capacity) {

       // Capacity too large, compute shrinking size

       shrink_bytes = capacity_until_GC - maximum_desired_capacity;

       // We don't want shrink all the way back to initSize if people call

       // System.gc(), because some programs do that between "phases" and then

       // we'd just have to grow the heap up again for the next phase.  So we

       // damp the shrinking: 0% on the first call, 10% on the second call, 40%

       // on the third call, and 100% by the fourth call.  But if we recompute

       // size without shrinking, it goes back to 0%.

       shrink_bytes = shrink_bytes / 100 * current_shrink_factor;

       shrink_bytes = align_size_down(shrink_bytes, Metaspace::commit_alignment());

       assert(shrink_bytes <= max_shrink_bytes,

         err_msg("invalid shrink size " SIZE_FORMAT " not <= " SIZE_FORMAT,

           shrink_bytes, max_shrink_bytes));

       if (current_shrink_factor == 0) {

         _shrink_factor = 10;

       } else {

         _shrink_factor = MIN2(current_shrink_factor * 4, (uint) 100);

       }

       if (PrintGCDetails && Verbose) {

         gclog_or_tty->print_cr("  "

                       "  shrinking:"

                       "  initSize: %.1fK"

                       "  maximum_desired_capacity: %.1fK",

                       MetaspaceSize / (double) K,

                       maximum_desired_capacity / (double) K);

         gclog_or_tty->print_cr("  "

                       "  shrink_bytes: %.1fK"

                       "  current_shrink_factor: %d"

                       "  new shrink factor: %d"

                       "  MinMetaspaceExpansion: %.1fK",

                       shrink_bytes / (double) K,

                       current_shrink_factor,

                       _shrink_factor,

                       MinMetaspaceExpansion / (double) K);

       }

     }

   }

   // Don't shrink unless it's significant

   if (shrink_bytes >= MinMetaspaceExpansion &&

       ((capacity_until_GC - shrink_bytes) >= MetaspaceSize)) {

     size_t new_capacity_until_GC = MetaspaceGC::dec_capacity_until_GC(shrink_bytes);

     Metaspace::tracer()->report_gc_threshold(capacity_until_GC,

                                              new_capacity_until_GC,

                                              MetaspaceGCThresholdUpdater::ComputeNewSize);

   }

 }

 // Metadebug methods

 void Metadebug::init_allocation_fail_alot_count() {

   if (MetadataAllocationFailALot) {

     _allocation_fail_alot_count =

       1+(long)((double)MetadataAllocationFailALotInterval*os::random()/(max_jint+1.0));

   }

 }

 #ifdef ASSERT

 bool Metadebug::test_metadata_failure() {

   if (MetadataAllocationFailALot &&

       Threads::is_vm_complete()) {

     if (_allocation_fail_alot_count > 0) {

       _allocation_fail_alot_count--;

     } else {

       if (TraceMetadataChunkAllocation && Verbose) {

         gclog_or_tty->print_cr("Metadata allocation failing for "

                                "MetadataAllocationFailALot");

       }

       init_allocation_fail_alot_count();

       return true;

     }

   }

   return false;

 }

 #endif

 // ChunkManager methods

 size_t ChunkManager::free_chunks_total_words() {

   return _free_chunks_total;

 }

 size_t ChunkManager::free_chunks_total_bytes() {

   return free_chunks_total_words() * BytesPerWord;

 }

 size_t ChunkManager::free_chunks_count() {

 #ifdef ASSERT

   if (!UseConcMarkSweepGC && !SpaceManager::expand_lock()->is_locked()) {

     MutexLockerEx cl(SpaceManager::expand_lock(),

                      Mutex::_no_safepoint_check_flag);

     // This lock is only needed in debug because the verification

     // of the _free_chunks_totals walks the list of free chunks

     slow_locked_verify_free_chunks_count();

   }

 #endif

   return _free_chunks_count;

 }

 void ChunkManager::locked_verify_free_chunks_total() {

   assert_lock_strong(SpaceManager::expand_lock());

   assert(sum_free_chunks() == _free_chunks_total,

     err_msg("_free_chunks_total " SIZE_FORMAT " is not the"

            " same as sum " SIZE_FORMAT, _free_chunks_total,

            sum_free_chunks()));

 }

 void ChunkManager::verify_free_chunks_total() {

   MutexLockerEx cl(SpaceManager::expand_lock(),

                      Mutex::_no_safepoint_check_flag);

   locked_verify_free_chunks_total();

 }

 void ChunkManager::locked_verify_free_chunks_count() {

   assert_lock_strong(SpaceManager::expand_lock());

   assert(sum_free_chunks_count() == _free_chunks_count,

     err_msg("_free_chunks_count " SIZE_FORMAT " is not the"

            " same as sum " SIZE_FORMAT, _free_chunks_count,

            sum_free_chunks_count()));

 }

 void ChunkManager::verify_free_chunks_count() {

 #ifdef ASSERT

   MutexLockerEx cl(SpaceManager::expand_lock(),

                      Mutex::_no_safepoint_check_flag);

   locked_verify_free_chunks_count();

 #endif

 }

 void ChunkManager::verify() {

   MutexLockerEx cl(SpaceManager::expand_lock(),

                      Mutex::_no_safepoint_check_flag);

   locked_verify();

 }

 void ChunkManager::locked_verify() {

   locked_verify_free_chunks_count();

   locked_verify_free_chunks_total();

 }

 void ChunkManager::locked_print_free_chunks(outputStream* st) {

   assert_lock_strong(SpaceManager::expand_lock());

   st->print_cr("Free chunk total " SIZE_FORMAT "  count " SIZE_FORMAT,

                 _free_chunks_total, _free_chunks_count);

 }

 void ChunkManager::locked_print_sum_free_chunks(outputStream* st) {

   assert_lock_strong(SpaceManager::expand_lock());

   st->print_cr("Sum free chunk total " SIZE_FORMAT "  count " SIZE_FORMAT,

                 sum_free_chunks(), sum_free_chunks_count());

 }

 ChunkList* ChunkManager::free_chunks(ChunkIndex index) {

   assert(index == SpecializedIndex || index == SmallIndex || index == MediumIndex,

          err_msg("Bad index: %d", (int)index));

   return &_free_chunks[index];

 }

 // These methods that sum the free chunk lists are used in printing

 // methods that are used in product builds.

 size_t ChunkManager::sum_free_chunks() {

   assert_lock_strong(SpaceManager::expand_lock());

   size_t result = 0;

   for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) {

     ChunkList* list = free_chunks(i);

     if (list == NULL) {

       continue;

     }

     result = result + list->count() * list->size();

   }

   result = result + humongous_dictionary()->total_size();

   return result;

 }

 size_t ChunkManager::sum_free_chunks_count() {

   assert_lock_strong(SpaceManager::expand_lock());

   size_t count = 0;

   for (ChunkIndex i = ZeroIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) {

     ChunkList* list = free_chunks(i);

     if (list == NULL) {

       continue;

     }

     count = count + list->count();

   }

   count = count + humongous_dictionary()->total_free_blocks();

   return count;

 }

 ChunkList* ChunkManager::find_free_chunks_list(size_t word_size) {

   ChunkIndex index = list_index(word_size);

   assert(index < HumongousIndex, "No humongous list");

   return free_chunks(index);

 }

 Metachunk* ChunkManager::free_chunks_get(size_t word_size) {

   assert_lock_strong(SpaceManager::expand_lock());

   slow_locked_verify();

   Metachunk* chunk = NULL;

   if (list_index(word_size) != HumongousIndex) {

     ChunkList* free_list = find_free_chunks_list(word_size);

     assert(free_list != NULL, "Sanity check");

     chunk = free_list->head();

     if (chunk == NULL) {

       return NULL;

     }

     // Remove the chunk as the head of the list.

     free_list->remove_chunk(chunk);

     if (TraceMetadataChunkAllocation && Verbose) {

       gclog_or_tty->print_cr("ChunkManager::free_chunks_get: free_list "

                              PTR_FORMAT " head " PTR_FORMAT " size " SIZE_FORMAT,

                              free_list, chunk, chunk->word_size());

     }

   } else {

     chunk = humongous_dictionary()->get_chunk(

       word_size,

       FreeBlockDictionary<Metachunk>::atLeast);

     if (chunk == NULL) {

       return NULL;

     }

     if (TraceMetadataHumongousAllocation) {

       size_t waste = chunk->word_size() - word_size;

       gclog_or_tty->print_cr("Free list allocate humongous chunk size "

                              SIZE_FORMAT " for requested size " SIZE_FORMAT

                              " waste " SIZE_FORMAT,

                              chunk->word_size(), word_size, waste);

     }

   }

   // Chunk is being removed from the chunks free list.

   dec_free_chunks_total(chunk->word_size());

   // Remove it from the links to this freelist

   chunk->set_next(NULL);

   chunk->set_prev(NULL);

 #ifdef ASSERT

   // Chunk is no longer on any freelist. Setting to false make container_count_slow()

   // work.

   chunk->set_is_tagged_free(false);

 #endif

   chunk->container()->inc_container_count();

   slow_locked_verify();

   return chunk;

 }

 Metachunk* ChunkManager::chunk_freelist_allocate(size_t word_size) {

   assert_lock_strong(SpaceManager::expand_lock());

   slow_locked_verify();

   // Take from the beginning of the list

   Metachunk* chunk = free_chunks_get(word_size);

   if (chunk == NULL) {

     return NULL;

   }

   assert((word_size <= chunk->word_size()) ||

          (list_index(chunk->word_size()) == HumongousIndex),

          "Non-humongous variable sized chunk");

   if (TraceMetadataChunkAllocation) {

     size_t list_count;

     if (list_index(word_size) < HumongousIndex) {

       ChunkList* list = find_free_chunks_list(word_size);

       list_count = list->count();

     } else {

       list_count = humongous_dictionary()->total_count();

     }

     gclog_or_tty->print("ChunkManager::chunk_freelist_allocate: " PTR_FORMAT " chunk "

                         PTR_FORMAT "  size " SIZE_FORMAT " count " SIZE_FORMAT " ",

                         this, chunk, chunk->word_size(), list_count);

     locked_print_free_chunks(gclog_or_tty);

   }

   return chunk;

 }

 void ChunkManager::print_on(outputStream* out) const {

   if (PrintFLSStatistics != 0) {

     const_cast<ChunkManager *>(this)->humongous_dictionary()->report_statistics();

   }

 }

 // SpaceManager methods

 size_t SpaceManager::adjust_initial_chunk_size(size_t requested, bool is_class_space) {

   size_t chunk_sizes[] = {

       specialized_chunk_size(is_class_space),

       small_chunk_size(is_class_space),

       medium_chunk_size(is_class_space)

   };

   // Adjust up to one of the fixed chunk sizes ...

   for (size_t i = 0; i < ARRAY_SIZE(chunk_sizes); i++) {

     if (requested <= chunk_sizes[i]) {

       return chunk_sizes[i];

     }

   }

   // ... or return the size as a humongous chunk.

   return requested;

 }

 size_t SpaceManager::adjust_initial_chunk_size(size_t requested) const {

   return adjust_initial_chunk_size(requested, is_class());

 }

 size_t SpaceManager::get_initial_chunk_size(Metaspace::MetaspaceType type) const {

   size_t requested;

   if (is_class()) {

     switch (type) {

     case Metaspace::BootMetaspaceType:       requested = Metaspace::first_class_chunk_word_size(); break;

     case Metaspace::ROMetaspaceType:         requested = ClassSpecializedChunk; break;

     case Metaspace::ReadWriteMetaspaceType:  requested = ClassSpecializedChunk; break;

     case Metaspace::AnonymousMetaspaceType:  requested = ClassSpecializedChunk; break;

     case Metaspace::ReflectionMetaspaceType: requested = ClassSpecializedChunk; break;

     default:                                 requested = ClassSmallChunk; break;

     }

   } else {

     switch (type) {

     case Metaspace::BootMetaspaceType:       requested = Metaspace::first_chunk_word_size(); break;

     case Metaspace::ROMetaspaceType:         requested = SharedReadOnlySize / wordSize; break;

     case Metaspace::ReadWriteMetaspaceType:  requested = SharedReadWriteSize / wordSize; break;

     case Metaspace::AnonymousMetaspaceType:  requested = SpecializedChunk; break;

     case Metaspace::ReflectionMetaspaceType: requested = SpecializedChunk; break;

     default:                                 requested = SmallChunk; break;

     }

   }

   // Adjust to one of the fixed chunk sizes (unless humongous)

   const size_t adjusted = adjust_initial_chunk_size(requested);

   assert(adjusted != 0, err_msg("Incorrect initial chunk size. Requested: "

          SIZE_FORMAT " adjusted: " SIZE_FORMAT, requested, adjusted));

   return adjusted;

 }

 size_t SpaceManager::sum_free_in_chunks_in_use() const {

   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);

   size_t free = 0;

   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {

     Metachunk* chunk = chunks_in_use(i);

     while (chunk != NULL) {

       free += chunk->free_word_size();

       chunk = chunk->next();

     }

   }

   return free;

 }

 size_t SpaceManager::sum_waste_in_chunks_in_use() const {

   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);

   size_t result = 0;

   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {

    result += sum_waste_in_chunks_in_use(i);

   }

   return result;

 }

 size_t SpaceManager::sum_waste_in_chunks_in_use(ChunkIndex index) const {

   size_t result = 0;

   Metachunk* chunk = chunks_in_use(index);

   // Count the free space in all the chunk but not the

   // current chunk from which allocations are still being done.

   while (chunk != NULL) {

     if (chunk != current_chunk()) {

       result += chunk->free_word_size();

     }

     chunk = chunk->next();

   }

   return result;

 }

 size_t SpaceManager::sum_capacity_in_chunks_in_use() const {

   // For CMS use "allocated_chunks_words()" which does not need the

   // Metaspace lock.  For the other collectors sum over the

   // lists.  Use both methods as a check that "allocated_chunks_words()"

   // is correct.  That is, sum_capacity_in_chunks() is too expensive

   // to use in the product and allocated_chunks_words() should be used

   // but allow for  checking that allocated_chunks_words() returns the same

   // value as sum_capacity_in_chunks_in_use() which is the definitive

   // answer.

   if (UseConcMarkSweepGC) {

     return allocated_chunks_words();

   } else {

     MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);

     size_t sum = 0;

     for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {

       Metachunk* chunk = chunks_in_use(i);

       while (chunk != NULL) {

         sum += chunk->word_size();

         chunk = chunk->next();

       }

     }

   return sum;

   }

 }

 size_t SpaceManager::sum_count_in_chunks_in_use() {

   size_t count = 0;

   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {

     count = count + sum_count_in_chunks_in_use(i);

   }

   return count;

 }

 size_t SpaceManager::sum_count_in_chunks_in_use(ChunkIndex i) {

   size_t count = 0;

   Metachunk* chunk = chunks_in_use(i);

   while (chunk != NULL) {

     count++;

     chunk = chunk->next();

   }

   return count;

 }

 size_t SpaceManager::sum_used_in_chunks_in_use() const {

   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);

   size_t used = 0;

   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {

     Metachunk* chunk = chunks_in_use(i);

     while (chunk != NULL) {

       used += chunk->used_word_size();

       chunk = chunk->next();

     }

   }

   return used;

 }

 void SpaceManager::locked_print_chunks_in_use_on(outputStream* st) const {

   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {

     Metachunk* chunk = chunks_in_use(i);

     st->print("SpaceManager: %s " PTR_FORMAT,

                  chunk_size_name(i), chunk);

     if (chunk != NULL) {

       st->print_cr(" free " SIZE_FORMAT,

                    chunk->free_word_size());

     } else {

       st->cr();

     }

   }

   chunk_manager()->locked_print_free_chunks(st);

   chunk_manager()->locked_print_sum_free_chunks(st);

 }

 size_t SpaceManager::calc_chunk_size(size_t word_size) {

   // Decide between a small chunk and a medium chunk.  Up to

   // _small_chunk_limit small chunks can be allocated but

   // once a medium chunk has been allocated, no more small

   // chunks will be allocated.

   size_t chunk_word_size;

   if (chunks_in_use(MediumIndex) == NULL &&

       sum_count_in_chunks_in_use(SmallIndex) < _small_chunk_limit) {

     chunk_word_size = (size_t) small_chunk_size();

     if (word_size + Metachunk::overhead() > small_chunk_size()) {

       chunk_word_size = medium_chunk_size();

     }

   } else {

     chunk_word_size = medium_chunk_size();

   }

   // Might still need a humongous chunk.  Enforce

   // humongous allocations sizes to be aligned up to

   // the smallest chunk size.

   size_t if_humongous_sized_chunk =

     align_size_up(word_size + Metachunk::overhead(),

                   smallest_chunk_size());

   chunk_word_size =

     MAX2((size_t) chunk_word_size, if_humongous_sized_chunk);

   assert(!SpaceManager::is_humongous(word_size) ||

          chunk_word_size == if_humongous_sized_chunk,

          err_msg("Size calculation is wrong, word_size " SIZE_FORMAT

                  " chunk_word_size " SIZE_FORMAT,

                  word_size, chunk_word_size));

   if (TraceMetadataHumongousAllocation &&

       SpaceManager::is_humongous(word_size)) {

     gclog_or_tty->print_cr("Metadata humongous allocation:");

     gclog_or_tty->print_cr("  word_size " PTR_FORMAT, word_size);

     gclog_or_tty->print_cr("  chunk_word_size " PTR_FORMAT,

                            chunk_word_size);

     gclog_or_tty->print_cr("    chunk overhead " PTR_FORMAT,

                            Metachunk::overhead());

   }

   return chunk_word_size;

 }

 void SpaceManager::track_metaspace_memory_usage() {

   if (is_init_completed()) {

     if (is_class()) {

       MemoryService::track_compressed_class_memory_usage();

     }

     MemoryService::track_metaspace_memory_usage();

   }

 }

 MetaWord* SpaceManager::grow_and_allocate(size_t word_size) {

   assert(vs_list()->current_virtual_space() != NULL,

          "Should have been set");

   assert(current_chunk() == NULL ||

          current_chunk()->allocate(word_size) == NULL,

          "Don't need to expand");

   MutexLockerEx cl(SpaceManager::expand_lock(), Mutex::_no_safepoint_check_flag);

   if (TraceMetadataChunkAllocation && Verbose) {

     size_t words_left = 0;

     size_t words_used = 0;

     if (current_chunk() != NULL) {

       words_left = current_chunk()->free_word_size();

       words_used = current_chunk()->used_word_size();

     }

     gclog_or_tty->print_cr("SpaceManager::grow_and_allocate for " SIZE_FORMAT

                            " words " SIZE_FORMAT " words used " SIZE_FORMAT

                            " words left",

                             word_size, words_used, words_left);

   }

   // Get another chunk out of the virtual space

   size_t chunk_word_size = calc_chunk_size(word_size);

   Metachunk* next = get_new_chunk(chunk_word_size);

   MetaWord* mem = NULL;

   // If a chunk was available, add it to the in-use chunk list

   // and do an allocation from it.

   if (next != NULL) {

     // Add to this manager's list of chunks in use.

     add_chunk(next, false);

     mem = next->allocate(word_size);

   }

   // Track metaspace memory usage statistic.

   track_metaspace_memory_usage();

   return mem;

 }

 void SpaceManager::print_on(outputStream* st) const {

   for (ChunkIndex i = ZeroIndex;

        i < NumberOfInUseLists ;

        i = next_chunk_index(i) ) {

     st->print_cr("  chunks_in_use " PTR_FORMAT " chunk size " PTR_FORMAT,

                  chunks_in_use(i),

                  chunks_in_use(i) == NULL ? 0 : chunks_in_use(i)->word_size());

   }

   st->print_cr("    waste:  Small " SIZE_FORMAT " Medium " SIZE_FORMAT

                " Humongous " SIZE_FORMAT,

                sum_waste_in_chunks_in_use(SmallIndex),

                sum_waste_in_chunks_in_use(MediumIndex),

                sum_waste_in_chunks_in_use(HumongousIndex));

   // block free lists

   if (block_freelists() != NULL) {

     st->print_cr("total in block free lists " SIZE_FORMAT,

       block_freelists()->total_size());

   }

 }

 SpaceManager::SpaceManager(Metaspace::MetadataType mdtype,

                            Mutex* lock) :

   _mdtype(mdtype),

   _allocated_blocks_words(0),

   _allocated_chunks_words(0),

   _allocated_chunks_count(0),

   _lock(lock)

 {

   initialize();

 }

 void SpaceManager::inc_size_metrics(size_t words) {

   assert_lock_strong(SpaceManager::expand_lock());

   // Total of allocated Metachunks and allocated Metachunks count

   // for each SpaceManager

   _allocated_chunks_words = _allocated_chunks_words + words;

   _allocated_chunks_count++;

   // Global total of capacity in allocated Metachunks

   MetaspaceAux::inc_capacity(mdtype(), words);

   // Global total of allocated Metablocks.

   // used_words_slow() includes the overhead in each

   // Metachunk so include it in the used when the

   // Metachunk is first added (so only added once per

   // Metachunk).

   MetaspaceAux::inc_used(mdtype(), Metachunk::overhead());

 }

 void SpaceManager::inc_used_metrics(size_t words) {

   // Add to the per SpaceManager total

   Atomic::add_ptr(words, &_allocated_blocks_words);

   // Add to the global total

   MetaspaceAux::inc_used(mdtype(), words);

 }

 void SpaceManager::dec_total_from_size_metrics() {

   MetaspaceAux::dec_capacity(mdtype(), allocated_chunks_words());

   MetaspaceAux::dec_used(mdtype(), allocated_blocks_words());

   // Also deduct the overhead per Metachunk

   MetaspaceAux::dec_used(mdtype(), allocated_chunks_count() * Metachunk::overhead());

 }

 void SpaceManager::initialize() {

   Metadebug::init_allocation_fail_alot_count();

   for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {

     _chunks_in_use[i] = NULL;

   }

   _current_chunk = NULL;

   if (TraceMetadataChunkAllocation && Verbose) {

     gclog_or_tty->print_cr("SpaceManager(): " PTR_FORMAT, this);

   }

 }

 void ChunkManager::return_chunks(ChunkIndex index, Metachunk* chunks) {

   if (chunks == NULL) {

     return;

   }

   ChunkList* list = free_chunks(index);

   assert(list->size() == chunks->word_size(), "Mismatch in chunk sizes");

   assert_lock_strong(SpaceManager::expand_lock());

   Metachunk* cur = chunks;

   // This returns chunks one at a time.  If a new

   // class List can be created that is a base class

   // of FreeList then something like FreeList::prepend()

   // can be used in place of this loop

   while (cur != NULL) {

     assert(cur->container() != NULL, "Container should have been set");

     cur->container()->dec_container_count();

     // Capture the next link before it is changed

     // by the call to return_chunk_at_head();

     Metachunk* next = cur->next();

     DEBUG_ONLY(cur->set_is_tagged_free(true);)

     list->return_chunk_at_head(cur);

     cur = next;

   }

 }

 SpaceManager::~SpaceManager() {

   // This call this->_lock which can't be done while holding expand_lock()

   assert(sum_capacity_in_chunks_in_use() == allocated_chunks_words(),

     err_msg("sum_capacity_in_chunks_in_use() " SIZE_FORMAT

             " allocated_chunks_words() " SIZE_FORMAT,

             sum_capacity_in_chunks_in_use(), allocated_chunks_words()));

   MutexLockerEx fcl(SpaceManager::expand_lock(),

                     Mutex::_no_safepoint_check_flag);

   chunk_manager()->slow_locked_verify();

   dec_total_from_size_metrics();

   if (TraceMetadataChunkAllocation && Verbose) {

     gclog_or_tty->print_cr("~SpaceManager(): " PTR_FORMAT, this);

     locked_print_chunks_in_use_on(gclog_or_tty);

   }

   // Do not mangle freed Metachunks.  The chunk size inside Metachunks

   // is during the freeing of a VirtualSpaceNodes.

   // Have to update before the chunks_in_use lists are emptied

   // below.

   chunk_manager()->inc_free_chunks_total(allocated_chunks_words(),

                                          sum_count_in_chunks_in_use());

   // Add all the chunks in use by this space manager

   // to the global list of free chunks.

   // Follow each list of chunks-in-use and add them to the

   // free lists.  Each list is NULL terminated.

   for (ChunkIndex i = ZeroIndex; i < HumongousIndex; i = next_chunk_index(i)) {

     if (TraceMetadataChunkAllocation && Verbose) {

       gclog_or_tty->print_cr("returned %d %s chunks to freelist",

                              sum_count_in_chunks_in_use(i),

                              chunk_size_name(i));

     }

     Metachunk* chunks = chunks_in_use(i);

     chunk_manager()->return_chunks(i, chunks);

     set_chunks_in_use(i, NULL);

     if (TraceMetadataChunkAllocation && Verbose) {

       gclog_or_tty->print_cr("updated freelist count %d %s",

                              chunk_manager()->free_chunks(i)->count(),

                              chunk_size_name(i));

     }

     assert(i != HumongousIndex, "Humongous chunks are handled explicitly later");

   }

   // The medium chunk case may be optimized by passing the head and

   // tail of the medium chunk list to add_at_head().  The tail is often

   // the current chunk but there are probably exceptions.

   // Humongous chunks

   if (TraceMetadataChunkAllocation && Verbose) {

     gclog_or_tty->print_cr("returned %d %s humongous chunks to dictionary",

                             sum_count_in_chunks_in_use(HumongousIndex),

                             chunk_size_name(HumongousIndex));

     gclog_or_tty->print("Humongous chunk dictionary: ");

   }

   // Humongous chunks are never the current chunk.

   Metachunk* humongous_chunks = chunks_in_use(HumongousIndex);

   while (humongous_chunks != NULL) {

 #ifdef ASSERT

     humongous_chunks->set_is_tagged_free(true);

 #endif

     if (TraceMetadataChunkAllocation && Verbose) {

       gclog_or_tty->print(PTR_FORMAT " (" SIZE_FORMAT ") ",

                           humongous_chunks,

                           humongous_chunks->word_size());

     }

     assert(humongous_chunks->word_size() == (size_t)

            align_size_up(humongous_chunks->word_size(),

                              smallest_chunk_size()),

            err_msg("Humongous chunk size is wrong: word size " SIZE_FORMAT

                    " granularity %d",

                    humongous_chunks->word_size(), smallest_chunk_size()));

     Metachunk* next_humongous_chunks = humongous_chunks->next();

     humongous_chunks->container()->dec_container_count();

     chunk_manager()->humongous_dictionary()->return_chunk(humongous_chunks);

     humongous_chunks = next_humongous_chunks;

   }

   if (TraceMetadataChunkAllocation && Verbose) {

     gclog_or_tty->cr();

     gclog_or_tty->print_cr("updated dictionary count %d %s",

                      chunk_manager()->humongous_dictionary()->total_count(),

                      chunk_size_name(HumongousIndex));

   }

   chunk_manager()->slow_locked_verify();

 }

 const char* SpaceManager::chunk_size_name(ChunkIndex index) const {

   switch (index) {

     case SpecializedIndex:

       return "Specialized";

     case SmallIndex:

       return "Small";

     case MediumIndex:

       return "Medium";

     case HumongousIndex:

       return "Humongous";

     default:

       return NULL;

   }

 }

 ChunkIndex ChunkManager::list_index(size_t size) {

   if (free_chunks(SpecializedIndex)->size() == size) {

     return SpecializedIndex;

   }

   if (free_chunks(SmallIndex)->size() == size) {

     return SmallIndex;

   }

   if (free_chunks(MediumIndex)->size() == size) {

     return MediumIndex;

   }

   assert(size > free_chunks(MediumIndex)->size(), "Not a humongous chunk");

   return HumongousIndex;

 }

 void SpaceManager::deallocate(MetaWord* p, size_t word_size) {

   assert_lock_strong(_lock);

   size_t raw_word_size = get_raw_word_size(word_size);

   size_t min_size = TreeChunk<Metablock, FreeList<Metablock> >::min_size();

   assert(raw_word_size >= min_size,

          err_msg("Should not deallocate dark matter " SIZE_FORMAT "<" SIZE_FORMAT, word_size, min_size));

   block_freelists()->return_block(p, raw_word_size);

 }

 // Adds a chunk to the list of chunks in use.

 void SpaceManager::add_chunk(Metachunk* new_chunk, bool make_current) {

   assert(new_chunk != NULL, "Should not be NULL");

   assert(new_chunk->next() == NULL, "Should not be on a list");

   new_chunk->reset_empty();

   // Find the correct list and and set the current

   // chunk for that list.

   ChunkIndex index = chunk_manager()->list_index(new_chunk->word_size());

   if (index != HumongousIndex) {

     retire_current_chunk();

     set_current_chunk(new_chunk);

     new_chunk->set_next(chunks_in_use(index));

     set_chunks_in_use(index, new_chunk);

   } else {

     // For null class loader data and DumpSharedSpaces, the first chunk isn't

     // small, so small will be null.  Link this first chunk as the current

     // chunk.

     if (make_current) {

       // Set as the current chunk but otherwise treat as a humongous chunk.

       set_current_chunk(new_chunk);

     }

     // Link at head.  The _current_chunk only points to a humongous chunk for

     // the null class loader metaspace (class and data virtual space managers)

     // any humongous chunks so will not point to the tail

     // of the humongous chunks list.

     new_chunk->set_next(chunks_in_use(HumongousIndex));

     set_chunks_in_use(HumongousIndex, new_chunk);

     assert(new_chunk->word_size() > medium_chunk_size(), "List inconsistency");

   }

   // Add to the running sum of capacity

   inc_size_metrics(new_chunk->word_size());

   assert(new_chunk->is_empty(), "Not ready for reuse");

   if (TraceMetadataChunkAllocation && Verbose) {

     gclog_or_tty->print("SpaceManager::add_chunk: %d) ",

                         sum_count_in_chunks_in_use());

     new_chunk->print_on(gclog_or_tty);

     chunk_manager()->locked_print_free_chunks(gclog_or_tty);

   }

 }

 void SpaceManager::retire_current_chunk() {

   if (current_chunk() != NULL) {

     size_t remaining_words = current_chunk()->free_word_size();

     if (remaining_words >= TreeChunk<Metablock, FreeList<Metablock> >::min_size()) {

       block_freelists()->return_block(current_chunk()->allocate(remaining_words), remaining_words);

       inc_used_metrics(remaining_words);

     }

   }

 }

 Metachunk* SpaceManager::get_new_chunk(size_t chunk_word_size) {

   // Get a chunk from the chunk freelist

   Metachunk* next = chunk_manager()->chunk_freelist_allocate(chunk_word_size);

   if (next == NULL) {

     next = vs_list()->get_new_chunk(chunk_word_size,

                                     medium_chunk_bunch());

   }

   if (TraceMetadataHumongousAllocation && next != NULL &&

       SpaceManager::is_humongous(next->word_size())) {

     gclog_or_tty->print_cr("  new humongous chunk word size "

                            PTR_FORMAT, next->word_size());

   }

   return next;

 }

 MetaWord* SpaceManager::allocate(size_t word_size) {

   MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag);

   size_t raw_word_size = get_raw_word_size(word_size);

   BlockFreelist* fl =  block_freelists();

   MetaWord* p = NULL;

   // Allocation from the dictionary is expensive in the sense that

   // the dictionary has to be searched for a size.  Don't allocate

   // from the dictionary until it starts to get fat.  Is this

   // a reasonable policy?  Maybe an skinny dictionary is fast enough

   // for allocations.  Do some profiling.  JJJ

   if (fl->total_size() > allocation_from_dictionary_limit) {

     p = fl->get_block(raw_word_size);

   }

   if (p == NULL) {

     p = allocate_work(raw_word_size);

   }

   return p;

 }

 // Returns the address of spaced allocated for "word_size".

 // This methods does not know about blocks (Metablocks)

 MetaWord* SpaceManager::allocate_work(size_t word_size) {

   assert_lock_strong(_lock);

 #ifdef ASSERT

   if (Metadebug::test_metadata_failure()) {

     return NULL;

   }

 #endif

   // Is there space in the current chunk?

   MetaWord* result = NULL;

   // For DumpSharedSpaces, only allocate out of the current chunk which is

   // never null because we gave it the size we wanted.   Caller reports out

   // of memory if this returns null.

   if (DumpSharedSpaces) {

     assert(current_chunk() != NULL, "should never happen");

     inc_used_metrics(word_size);

     return current_chunk()->allocate(word_size); // caller handles null result

   }

   if (current_chunk() != NULL) {

     result = current_chunk()->allocate(word_size);

   }

   if (result == NULL) {

     result = grow_and_allocate(word_size);

   }

   if (result != NULL) {

     inc_used_metrics(word_size);

     assert(result != (MetaWord*) chunks_in_use(MediumIndex),

            "Head of the list is being allocated");

   }

   return result;

 }

 void SpaceManager::verify() {

   // If there are blocks in the dictionary, then

   // verfication of chunks does not work since

   // being in the dictionary alters a chunk.

   if (block_freelists()->total_size() == 0) {

     for (ChunkIndex i = ZeroIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) {

       Metachunk* curr = chunks_in_use(i);

       while (curr != NULL) {

         curr->verify();

         verify_chunk_size(curr);

         curr = curr->next();

       }

     }

   }

 }

 void SpaceManager::verify_chunk_size(Metachunk* chunk) {

   assert(is_humongous(chunk->word_size()) ||

          chunk->word_size() == medium_chunk_size() ||

          chunk->word_size() == small_chunk_size() ||

          chunk->word_size() == specialized_chunk_size(),

          "Chunk size is wrong");

   return;

 }

 #ifdef ASSERT

 void SpaceManager::verify_allocated_blocks_words() {

   // Verification is only guaranteed at a safepoint.

   assert(SafepointSynchronize::is_at_safepoint() || !Universe::is_fully_initialized(),

     "Verification can fail if the applications is running");

   assert(allocated_blocks_words() == sum_used_in_chunks_in_use(),

     err_msg("allocation total is not consistent " SIZE_FORMAT

             " vs " SIZE_FORMAT,

             allocated_blocks_words(), sum_used_in_chunks_in_use()));

 }

 #endif

 void SpaceManager::dump(outputStream* const out) const {

   size_t curr_total = 0;

   size_t waste = 0;

   uint i = 0;

   size_t used = 0;

   size_t capacity = 0;

   // Add up statistics for all chunks in this SpaceManager.

   for (ChunkIndex index = ZeroIndex;

        index < NumberOfInUseLists;

        index = next_chunk_index(index)) {

     for (Metachunk* curr = chunks_in_use(index);

          curr != NULL;

          curr = curr->next()) {

       out->print("%d) ", i++);

       curr->print_on(out);

       curr_total += curr->word_size();

       used += curr->used_word_size();

       capacity += curr->word_size();

       waste += curr->free_word_size() + curr->overhead();;

     }

   }

   if (TraceMetadataChunkAllocation && Verbose) {

     block_freelists()->print_on(out);

   }

   size_t free = current_chunk() == NULL ? 0 : current_chunk()->free_word_size();

   // Free space isn't wasted.

   waste -= free;

   out->print_cr("total of all chunks "  SIZE_FORMAT " used " SIZE_FORMAT

                 " free " SIZE_FORMAT " capacity " SIZE_FORMAT

                 " waste " SIZE_FORMAT, curr_total, used, free, capacity, waste);

 }

 #ifndef PRODUCT

 void SpaceManager::mangle_freed_chunks() {

   for (ChunkIndex index = ZeroIndex;

        index < NumberOfInUseLists;

        index = next_chunk_index(index)) {

     for (Metachunk* curr = chunks_in_use(index);

          curr != NULL;

          curr = curr->next()) {

       curr->mangle();

     }

   }

 }

 #endif // PRODUCT

 // MetaspaceAux

 size_t MetaspaceAux::_capacity_words[] = {0, 0};

 size_t MetaspaceAux::_used_words[] = {0, 0};

 size_t MetaspaceAux::free_bytes(Metaspace::MetadataType mdtype) {

   VirtualSpaceList* list = Metaspace::get_space_list(mdtype);

   return list == NULL ? 0 : list->free_bytes();

 }

 size_t MetaspaceAux::free_bytes() {

   return free_bytes(Metaspace::ClassType) + free_bytes(Metaspace::NonClassType);

 }

 void MetaspaceAux::dec_capacity(Metaspace::MetadataType mdtype, size_t words) {