duke@435: /* duke@435: * Copyright 2006-2007 Sun Microsystems, Inc. All Rights Reserved. duke@435: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. duke@435: * duke@435: * This code is free software; you can redistribute it and/or modify it duke@435: * under the terms of the GNU General Public License version 2 only, as duke@435: * published by the Free Software Foundation. duke@435: * duke@435: * This code is distributed in the hope that it will be useful, but WITHOUT duke@435: * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or duke@435: * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License duke@435: * version 2 for more details (a copy is included in the LICENSE file that duke@435: * accompanied this code). duke@435: * duke@435: * You should have received a copy of the GNU General Public License version duke@435: * 2 along with this work; if not, write to the Free Software Foundation, duke@435: * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. duke@435: * duke@435: * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, duke@435: * CA 95054 USA or visit www.sun.com if you need additional information or duke@435: * have any questions. duke@435: * duke@435: */ duke@435: duke@435: /* duke@435: * The NUMA-aware allocator (MutableNUMASpace) is basically a modification duke@435: * of MutableSpace which preserves interfaces but implements different duke@435: * functionality. The space is split into chunks for each locality group duke@435: * (resizing for adaptive size policy is also supported). For each thread duke@435: * allocations are performed in the chunk corresponding to the home locality duke@435: * group of the thread. Whenever any chunk fills-in the young generation duke@435: * collection occurs. duke@435: * The chunks can be also be adaptively resized. The idea behind the adaptive duke@435: * sizing is to reduce the loss of the space in the eden due to fragmentation. duke@435: * The main cause of fragmentation is uneven allocation rates of threads. duke@435: * The allocation rate difference between locality groups may be caused either by duke@435: * application specifics or by uneven LWP distribution by the OS. Besides, duke@435: * application can have less threads then the number of locality groups. duke@435: * In order to resize the chunk we measure the allocation rate of the duke@435: * application between collections. After that we reshape the chunks to reflect duke@435: * the allocation rate pattern. The AdaptiveWeightedAverage exponentially duke@435: * decaying average is used to smooth the measurements. The NUMASpaceResizeRate duke@435: * parameter is used to control the adaptation speed by restricting the number of duke@435: * bytes that can be moved during the adaptation phase. duke@435: * Chunks may contain pages from a wrong locality group. The page-scanner has duke@435: * been introduced to address the problem. Remote pages typically appear due to duke@435: * the memory shortage in the target locality group. Besides Solaris would duke@435: * allocate a large page from the remote locality group even if there are small duke@435: * local pages available. The page-scanner scans the pages right after the duke@435: * collection and frees remote pages in hope that subsequent reallocation would duke@435: * be more successful. This approach proved to be useful on systems with high duke@435: * load where multiple processes are competing for the memory. duke@435: */ duke@435: duke@435: class MutableNUMASpace : public MutableSpace { duke@435: friend class VMStructs; duke@435: duke@435: class LGRPSpace : public CHeapObj { duke@435: int _lgrp_id; duke@435: MutableSpace* _space; duke@435: MemRegion _invalid_region; duke@435: AdaptiveWeightedAverage *_alloc_rate; duke@435: duke@435: struct SpaceStats { duke@435: size_t _local_space, _remote_space, _unbiased_space, _uncommited_space; duke@435: size_t _large_pages, _small_pages; duke@435: duke@435: SpaceStats() { duke@435: _local_space = 0; duke@435: _remote_space = 0; duke@435: _unbiased_space = 0; duke@435: _uncommited_space = 0; duke@435: _large_pages = 0; duke@435: _small_pages = 0; duke@435: } duke@435: }; duke@435: duke@435: SpaceStats _space_stats; duke@435: duke@435: char* _last_page_scanned; duke@435: char* last_page_scanned() { return _last_page_scanned; } duke@435: void set_last_page_scanned(char* p) { _last_page_scanned = p; } duke@435: public: duke@435: LGRPSpace(int l) : _lgrp_id(l), _last_page_scanned(NULL) { duke@435: _space = new MutableSpace(); duke@435: _alloc_rate = new AdaptiveWeightedAverage(NUMAChunkResizeWeight); duke@435: } duke@435: ~LGRPSpace() { duke@435: delete _space; duke@435: delete _alloc_rate; duke@435: } duke@435: duke@435: void add_invalid_region(MemRegion r) { duke@435: if (!_invalid_region.is_empty()) { duke@435: _invalid_region.set_start(MIN2(_invalid_region.start(), r.start())); duke@435: _invalid_region.set_end(MAX2(_invalid_region.end(), r.end())); duke@435: } else { duke@435: _invalid_region = r; duke@435: } duke@435: } duke@435: duke@435: static bool equals(void* lgrp_id_value, LGRPSpace* p) { duke@435: return *(int*)lgrp_id_value == p->lgrp_id(); duke@435: } duke@435: duke@435: void sample() { duke@435: alloc_rate()->sample(space()->used_in_bytes()); duke@435: } duke@435: duke@435: MemRegion invalid_region() const { return _invalid_region; } duke@435: void set_invalid_region(MemRegion r) { _invalid_region = r; } duke@435: int lgrp_id() const { return _lgrp_id; } duke@435: MutableSpace* space() const { return _space; } duke@435: AdaptiveWeightedAverage* alloc_rate() const { return _alloc_rate; } duke@435: SpaceStats* space_stats() { return &_space_stats; } duke@435: void clear_space_stats() { _space_stats = SpaceStats(); } duke@435: duke@435: void accumulate_statistics(size_t page_size); duke@435: void scan_pages(size_t page_size, size_t page_count); duke@435: }; duke@435: duke@435: GrowableArray* _lgrp_spaces; duke@435: size_t _page_size; duke@435: unsigned _adaptation_cycles, _samples_count; duke@435: duke@435: void set_page_size(size_t psz) { _page_size = psz; } duke@435: size_t page_size() const { return _page_size; } duke@435: duke@435: unsigned adaptation_cycles() { return _adaptation_cycles; } duke@435: void set_adaptation_cycles(int v) { _adaptation_cycles = v; } duke@435: duke@435: unsigned samples_count() { return _samples_count; } duke@435: void increment_samples_count() { ++_samples_count; } duke@435: duke@435: size_t _base_space_size; duke@435: void set_base_space_size(size_t v) { _base_space_size = v; } duke@435: size_t base_space_size() const { return _base_space_size; } duke@435: duke@435: // Check if the NUMA topology has changed. Add and remove spaces if needed. duke@435: // The update can be forced by setting the force parameter equal to true. duke@435: bool update_layout(bool force); duke@435: // Bias region towards the first-touching lgrp. duke@435: void bias_region(MemRegion mr); duke@435: // Free pages in a given region. duke@435: void free_region(MemRegion mr); duke@435: // Get current chunk size. duke@435: size_t current_chunk_size(int i); duke@435: // Get default chunk size (equally divide the space). duke@435: size_t default_chunk_size(); duke@435: // Adapt the chunk size to follow the allocation rate. duke@435: size_t adaptive_chunk_size(int i, size_t limit); duke@435: // Scan and free invalid pages. duke@435: void scan_pages(size_t page_count); duke@435: // Return the bottom_region and the top_region. Align them to page_size() boundary. duke@435: // |------------------new_region---------------------------------| duke@435: // |----bottom_region--|---intersection---|------top_region------| duke@435: void select_tails(MemRegion new_region, MemRegion intersection, duke@435: MemRegion* bottom_region, MemRegion *top_region); duke@435: // Try to merge the invalid region with the bottom or top region by decreasing duke@435: // the intersection area. Return the invalid_region aligned to the page_size() duke@435: // boundary if it's inside the intersection. Return non-empty invalid_region duke@435: // if it lies inside the intersection (also page-aligned). duke@435: // |------------------new_region---------------------------------| duke@435: // |----------------|-------invalid---|--------------------------| duke@435: // |----bottom_region--|---intersection---|------top_region------| duke@435: void merge_regions(MemRegion new_region, MemRegion* intersection, duke@435: MemRegion *invalid_region); duke@435: duke@435: public: duke@435: GrowableArray* lgrp_spaces() const { return _lgrp_spaces; } duke@435: MutableNUMASpace(); duke@435: virtual ~MutableNUMASpace(); duke@435: // Space initialization. duke@435: virtual void initialize(MemRegion mr, bool clear_space); duke@435: // Update space layout if necessary. Do all adaptive resizing job. duke@435: virtual void update(); duke@435: // Update allocation rate averages. duke@435: virtual void accumulate_statistics(); duke@435: duke@435: virtual void clear(); duke@435: virtual void mangle_unused_area(); duke@435: virtual void ensure_parsability(); duke@435: virtual size_t used_in_words() const; duke@435: virtual size_t free_in_words() const; duke@435: virtual size_t tlab_capacity(Thread* thr) const; duke@435: virtual size_t unsafe_max_tlab_alloc(Thread* thr) const; duke@435: duke@435: // Allocation (return NULL if full) duke@435: virtual HeapWord* allocate(size_t word_size); duke@435: virtual HeapWord* cas_allocate(size_t word_size); duke@435: duke@435: // Debugging duke@435: virtual void print_on(outputStream* st) const; duke@435: virtual void print_short_on(outputStream* st) const; duke@435: virtual void verify(bool allow_dirty) const; duke@435: duke@435: virtual void set_top(HeapWord* value); duke@435: };