duke@435: duke@435: /* drchase@6680: * Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. duke@435: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. duke@435: * duke@435: * This code is free software; you can redistribute it and/or modify it duke@435: * under the terms of the GNU General Public License version 2 only, as duke@435: * published by the Free Software Foundation. duke@435: * duke@435: * This code is distributed in the hope that it will be useful, but WITHOUT duke@435: * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or duke@435: * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License duke@435: * version 2 for more details (a copy is included in the LICENSE file that duke@435: * accompanied this code). duke@435: * duke@435: * You should have received a copy of the GNU General Public License version duke@435: * 2 along with this work; if not, write to the Free Software Foundation, duke@435: * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. duke@435: * trims@1907: * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA trims@1907: * or visit www.oracle.com if you need additional information or have any trims@1907: * questions. duke@435: * duke@435: */ duke@435: stefank@2314: #include "precompiled.hpp" stefank@2314: #include "gc_implementation/shared/mutableNUMASpace.hpp" stefank@2314: #include "gc_implementation/shared/spaceDecorator.hpp" stefank@2314: #include "memory/sharedHeap.hpp" stefank@2314: #include "oops/oop.inline.hpp" stefank@4299: #include "runtime/thread.inline.hpp" duke@435: drchase@6680: PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC drchase@6680: iveresov@970: MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) { zgu@3900: _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray(0, true); duke@435: _page_size = os::vm_page_size(); duke@435: _adaptation_cycles = 0; duke@435: _samples_count = 0; duke@435: update_layout(true); duke@435: } duke@435: duke@435: MutableNUMASpace::~MutableNUMASpace() { duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: delete lgrp_spaces()->at(i); duke@435: } duke@435: delete lgrp_spaces(); duke@435: } duke@435: jmasa@698: #ifndef PRODUCT duke@435: void MutableNUMASpace::mangle_unused_area() { jmasa@698: // This method should do nothing. jmasa@698: // It can be called on a numa space during a full compaction. duke@435: } jmasa@698: void MutableNUMASpace::mangle_unused_area_complete() { jmasa@698: // This method should do nothing. jmasa@698: // It can be called on a numa space during a full compaction. jmasa@698: } jmasa@698: void MutableNUMASpace::mangle_region(MemRegion mr) { jmasa@698: // This method should do nothing because numa spaces are not mangled. jmasa@698: } jmasa@698: void MutableNUMASpace::set_top_for_allocations(HeapWord* v) { jmasa@698: assert(false, "Do not mangle MutableNUMASpace's"); jmasa@698: } jmasa@698: void MutableNUMASpace::set_top_for_allocations() { jmasa@698: // This method should do nothing. jmasa@698: } jmasa@698: void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) { jmasa@698: // This method should do nothing. jmasa@698: } jmasa@698: void MutableNUMASpace::check_mangled_unused_area_complete() { jmasa@698: // This method should do nothing. jmasa@698: } jmasa@698: #endif // NOT_PRODUCT duke@435: duke@435: // There may be unallocated holes in the middle chunks duke@435: // that should be filled with dead objects to ensure parseability. duke@435: void MutableNUMASpace::ensure_parsability() { duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: LGRPSpace *ls = lgrp_spaces()->at(i); duke@435: MutableSpace *s = ls->space(); twisti@1040: if (s->top() < top()) { // For all spaces preceding the one containing top() duke@435: if (s->free_in_words() > 0) { brutisso@3668: intptr_t cur_top = (intptr_t)s->top(); brutisso@3668: size_t words_left_to_fill = pointer_delta(s->end(), s->top());; brutisso@3668: while (words_left_to_fill > 0) { brutisso@3668: size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size()); brutisso@3668: assert(words_to_fill >= CollectedHeap::min_fill_size(), kevinw@9327: err_msg("Remaining size (" SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")", brutisso@3668: words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size())); brutisso@3668: CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill); brutisso@3668: if (!os::numa_has_static_binding()) { brutisso@3668: size_t touched_words = words_to_fill; duke@435: #ifndef ASSERT brutisso@3668: if (!ZapUnusedHeapArea) { brutisso@3668: touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)), brutisso@3668: touched_words); brutisso@3668: } duke@435: #endif brutisso@3668: MemRegion invalid; brutisso@3668: HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size()); brutisso@3668: HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size()); brutisso@3668: if (crossing_start != crossing_end) { brutisso@3668: // If object header crossed a small page boundary we mark the area brutisso@3668: // as invalid rounding it to a page_size(). brutisso@3668: HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom()); brutisso@3668: HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end()); brutisso@3668: invalid = MemRegion(start, end); brutisso@3668: } brutisso@3668: brutisso@3668: ls->add_invalid_region(invalid); iveresov@576: } brutisso@3668: cur_top = cur_top + (words_to_fill * HeapWordSize); brutisso@3668: words_left_to_fill -= words_to_fill; duke@435: } duke@435: } duke@435: } else { iveresov@576: if (!os::numa_has_static_binding()) { duke@435: #ifdef ASSERT duke@435: MemRegion invalid(s->top(), s->end()); duke@435: ls->add_invalid_region(invalid); iveresov@576: #else iveresov@576: if (ZapUnusedHeapArea) { iveresov@576: MemRegion invalid(s->top(), s->end()); iveresov@576: ls->add_invalid_region(invalid); iveresov@579: } else { iveresov@579: return; iveresov@579: } duke@435: #endif iveresov@579: } else { iveresov@579: return; iveresov@576: } duke@435: } duke@435: } duke@435: } duke@435: duke@435: size_t MutableNUMASpace::used_in_words() const { duke@435: size_t s = 0; duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: s += lgrp_spaces()->at(i)->space()->used_in_words(); duke@435: } duke@435: return s; duke@435: } duke@435: duke@435: size_t MutableNUMASpace::free_in_words() const { duke@435: size_t s = 0; duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: s += lgrp_spaces()->at(i)->space()->free_in_words(); duke@435: } duke@435: return s; duke@435: } duke@435: duke@435: duke@435: size_t MutableNUMASpace::tlab_capacity(Thread *thr) const { duke@435: guarantee(thr != NULL, "No thread"); duke@435: int lgrp_id = thr->lgrp_id(); iveresov@703: if (lgrp_id == -1) { iveresov@703: // This case can occur after the topology of the system has iveresov@703: // changed. Thread can change their location, the new home iveresov@703: // group will be determined during the first allocation iveresov@703: // attempt. For now we can safely assume that all spaces iveresov@703: // have equal size because the whole space will be reinitialized. iveresov@703: if (lgrp_spaces()->length() > 0) { iveresov@703: return capacity_in_bytes() / lgrp_spaces()->length(); iveresov@703: } else { iveresov@703: assert(false, "There should be at least one locality group"); iveresov@703: return 0; iveresov@703: } iveresov@703: } iveresov@703: // That's the normal case, where we know the locality group of the thread. duke@435: int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); duke@435: if (i == -1) { duke@435: return 0; duke@435: } duke@435: return lgrp_spaces()->at(i)->space()->capacity_in_bytes(); duke@435: } duke@435: brutisso@6376: size_t MutableNUMASpace::tlab_used(Thread *thr) const { brutisso@6376: // Please see the comments for tlab_capacity(). brutisso@6376: guarantee(thr != NULL, "No thread"); brutisso@6376: int lgrp_id = thr->lgrp_id(); brutisso@6376: if (lgrp_id == -1) { brutisso@6376: if (lgrp_spaces()->length() > 0) { brutisso@6376: return (used_in_bytes()) / lgrp_spaces()->length(); brutisso@6376: } else { brutisso@6376: assert(false, "There should be at least one locality group"); brutisso@6376: return 0; brutisso@6376: } brutisso@6376: } brutisso@6376: int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); brutisso@6376: if (i == -1) { brutisso@6376: return 0; brutisso@6376: } brutisso@6376: return lgrp_spaces()->at(i)->space()->used_in_bytes(); brutisso@6376: } brutisso@6376: brutisso@6376: duke@435: size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const { iveresov@703: // Please see the comments for tlab_capacity(). duke@435: guarantee(thr != NULL, "No thread"); duke@435: int lgrp_id = thr->lgrp_id(); iveresov@703: if (lgrp_id == -1) { iveresov@703: if (lgrp_spaces()->length() > 0) { iveresov@703: return free_in_bytes() / lgrp_spaces()->length(); iveresov@703: } else { iveresov@703: assert(false, "There should be at least one locality group"); iveresov@703: return 0; iveresov@703: } iveresov@703: } duke@435: int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); duke@435: if (i == -1) { duke@435: return 0; duke@435: } duke@435: return lgrp_spaces()->at(i)->space()->free_in_bytes(); duke@435: } duke@435: iveresov@808: iveresov@808: size_t MutableNUMASpace::capacity_in_words(Thread* thr) const { iveresov@808: guarantee(thr != NULL, "No thread"); iveresov@808: int lgrp_id = thr->lgrp_id(); iveresov@808: if (lgrp_id == -1) { iveresov@808: if (lgrp_spaces()->length() > 0) { iveresov@808: return capacity_in_words() / lgrp_spaces()->length(); iveresov@808: } else { iveresov@808: assert(false, "There should be at least one locality group"); iveresov@808: return 0; iveresov@808: } iveresov@808: } iveresov@808: int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); iveresov@808: if (i == -1) { iveresov@808: return 0; iveresov@808: } iveresov@808: return lgrp_spaces()->at(i)->space()->capacity_in_words(); iveresov@808: } iveresov@808: duke@435: // Check if the NUMA topology has changed. Add and remove spaces if needed. duke@435: // The update can be forced by setting the force parameter equal to true. duke@435: bool MutableNUMASpace::update_layout(bool force) { duke@435: // Check if the topology had changed. duke@435: bool changed = os::numa_topology_changed(); duke@435: if (force || changed) { duke@435: // Compute lgrp intersection. Add/remove spaces. duke@435: int lgrp_limit = (int)os::numa_get_groups_num(); zgu@3900: int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC); duke@435: int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit); duke@435: assert(lgrp_num > 0, "There should be at least one locality group"); duke@435: // Add new spaces for the new nodes duke@435: for (int i = 0; i < lgrp_num; i++) { duke@435: bool found = false; duke@435: for (int j = 0; j < lgrp_spaces()->length(); j++) { duke@435: if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) { duke@435: found = true; duke@435: break; duke@435: } duke@435: } duke@435: if (!found) { iveresov@970: lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment())); duke@435: } duke@435: } duke@435: duke@435: // Remove spaces for the removed nodes. duke@435: for (int i = 0; i < lgrp_spaces()->length();) { duke@435: bool found = false; duke@435: for (int j = 0; j < lgrp_num; j++) { duke@435: if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) { duke@435: found = true; duke@435: break; duke@435: } duke@435: } duke@435: if (!found) { duke@435: delete lgrp_spaces()->at(i); duke@435: lgrp_spaces()->remove_at(i); duke@435: } else { duke@435: i++; duke@435: } duke@435: } duke@435: zgu@3900: FREE_C_HEAP_ARRAY(int, lgrp_ids, mtGC); duke@435: duke@435: if (changed) { duke@435: for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) { duke@435: thread->set_lgrp_id(-1); duke@435: } duke@435: } duke@435: return true; duke@435: } duke@435: return false; duke@435: } duke@435: duke@435: // Bias region towards the first-touching lgrp. Set the right page sizes. iveresov@576: void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) { duke@435: HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); duke@435: HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); duke@435: if (end > start) { duke@435: MemRegion aligned_region(start, end); duke@435: assert((intptr_t)aligned_region.start() % page_size() == 0 && duke@435: (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); duke@435: assert(region().contains(aligned_region), "Sanity"); iveresov@576: // First we tell the OS which page size we want in the given range. The underlying iveresov@576: // large page can be broken down if we require small pages. iveresov@576: os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); iveresov@576: // Then we uncommit the pages in the range. iveresov@3363: os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); iveresov@576: // And make them local/first-touch biased. iveresov@576: os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id); duke@435: } duke@435: } duke@435: duke@435: // Free all pages in the region. duke@435: void MutableNUMASpace::free_region(MemRegion mr) { duke@435: HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); duke@435: HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); duke@435: if (end > start) { duke@435: MemRegion aligned_region(start, end); duke@435: assert((intptr_t)aligned_region.start() % page_size() == 0 && duke@435: (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); duke@435: assert(region().contains(aligned_region), "Sanity"); iveresov@3363: os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); duke@435: } duke@435: } duke@435: duke@435: // Update space layout. Perform adaptation. duke@435: void MutableNUMASpace::update() { duke@435: if (update_layout(false)) { duke@435: // If the topology has changed, make all chunks zero-sized. iveresov@703: // And clear the alloc-rate statistics. iveresov@703: // In future we may want to handle this more gracefully in order iveresov@703: // to avoid the reallocation of the pages as much as possible. duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { iveresov@703: LGRPSpace *ls = lgrp_spaces()->at(i); iveresov@703: MutableSpace *s = ls->space(); duke@435: s->set_end(s->bottom()); duke@435: s->set_top(s->bottom()); iveresov@703: ls->clear_alloc_rate(); duke@435: } jmasa@698: // A NUMA space is never mangled jmasa@698: initialize(region(), jmasa@698: SpaceDecorator::Clear, jmasa@698: SpaceDecorator::DontMangle); duke@435: } else { duke@435: bool should_initialize = false; iveresov@576: if (!os::numa_has_static_binding()) { iveresov@576: for (int i = 0; i < lgrp_spaces()->length(); i++) { iveresov@576: if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) { iveresov@576: should_initialize = true; iveresov@576: break; iveresov@576: } duke@435: } duke@435: } duke@435: duke@435: if (should_initialize || duke@435: (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) { jmasa@698: // A NUMA space is never mangled jmasa@698: initialize(region(), jmasa@698: SpaceDecorator::Clear, jmasa@698: SpaceDecorator::DontMangle); duke@435: } duke@435: } duke@435: duke@435: if (NUMAStats) { duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: lgrp_spaces()->at(i)->accumulate_statistics(page_size()); duke@435: } duke@435: } duke@435: duke@435: scan_pages(NUMAPageScanRate); duke@435: } duke@435: duke@435: // Scan pages. Free pages that have smaller size or wrong placement. duke@435: void MutableNUMASpace::scan_pages(size_t page_count) duke@435: { duke@435: size_t pages_per_chunk = page_count / lgrp_spaces()->length(); duke@435: if (pages_per_chunk > 0) { duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: LGRPSpace *ls = lgrp_spaces()->at(i); duke@435: ls->scan_pages(page_size(), pages_per_chunk); duke@435: } duke@435: } duke@435: } duke@435: duke@435: // Accumulate statistics about the allocation rate of each lgrp. duke@435: void MutableNUMASpace::accumulate_statistics() { duke@435: if (UseAdaptiveNUMAChunkSizing) { duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: lgrp_spaces()->at(i)->sample(); duke@435: } duke@435: increment_samples_count(); duke@435: } duke@435: duke@435: if (NUMAStats) { duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: lgrp_spaces()->at(i)->accumulate_statistics(page_size()); duke@435: } duke@435: } duke@435: } duke@435: duke@435: // Get the current size of a chunk. duke@435: // This function computes the size of the chunk based on the duke@435: // difference between chunk ends. This allows it to work correctly in duke@435: // case the whole space is resized and during the process of adaptive duke@435: // chunk resizing. duke@435: size_t MutableNUMASpace::current_chunk_size(int i) { duke@435: HeapWord *cur_end, *prev_end; duke@435: if (i == 0) { duke@435: prev_end = bottom(); duke@435: } else { duke@435: prev_end = lgrp_spaces()->at(i - 1)->space()->end(); duke@435: } duke@435: if (i == lgrp_spaces()->length() - 1) { duke@435: cur_end = end(); duke@435: } else { duke@435: cur_end = lgrp_spaces()->at(i)->space()->end(); duke@435: } duke@435: if (cur_end > prev_end) { duke@435: return pointer_delta(cur_end, prev_end, sizeof(char)); duke@435: } duke@435: return 0; duke@435: } duke@435: duke@435: // Return the default chunk size by equally diving the space. duke@435: // page_size() aligned. duke@435: size_t MutableNUMASpace::default_chunk_size() { duke@435: return base_space_size() / lgrp_spaces()->length() * page_size(); duke@435: } duke@435: duke@435: // Produce a new chunk size. page_size() aligned. iveresov@826: // This function is expected to be called on sequence of i's from 0 to iveresov@826: // lgrp_spaces()->length(). duke@435: size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) { duke@435: size_t pages_available = base_space_size(); duke@435: for (int j = 0; j < i; j++) { duke@435: pages_available -= round_down(current_chunk_size(j), page_size()) / page_size(); duke@435: } duke@435: pages_available -= lgrp_spaces()->length() - i - 1; duke@435: assert(pages_available > 0, "No pages left"); duke@435: float alloc_rate = 0; duke@435: for (int j = i; j < lgrp_spaces()->length(); j++) { duke@435: alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average(); duke@435: } duke@435: size_t chunk_size = 0; duke@435: if (alloc_rate > 0) { duke@435: LGRPSpace *ls = lgrp_spaces()->at(i); iveresov@826: chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size(); duke@435: } duke@435: chunk_size = MAX2(chunk_size, page_size()); duke@435: duke@435: if (limit > 0) { duke@435: limit = round_down(limit, page_size()); duke@435: if (chunk_size > current_chunk_size(i)) { iveresov@897: size_t upper_bound = pages_available * page_size(); iveresov@897: if (upper_bound > limit && iveresov@897: current_chunk_size(i) < upper_bound - limit) { iveresov@897: // The resulting upper bound should not exceed the available iveresov@897: // amount of memory (pages_available * page_size()). iveresov@897: upper_bound = current_chunk_size(i) + limit; iveresov@897: } iveresov@897: chunk_size = MIN2(chunk_size, upper_bound); duke@435: } else { iveresov@897: size_t lower_bound = page_size(); iveresov@897: if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow. iveresov@897: lower_bound = current_chunk_size(i) - limit; iveresov@897: } iveresov@897: chunk_size = MAX2(chunk_size, lower_bound); duke@435: } duke@435: } duke@435: assert(chunk_size <= pages_available * page_size(), "Chunk size out of range"); duke@435: return chunk_size; duke@435: } duke@435: duke@435: duke@435: // Return the bottom_region and the top_region. Align them to page_size() boundary. duke@435: // |------------------new_region---------------------------------| duke@435: // |----bottom_region--|---intersection---|------top_region------| duke@435: void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection, duke@435: MemRegion* bottom_region, MemRegion *top_region) { duke@435: // Is there bottom? duke@435: if (new_region.start() < intersection.start()) { // Yes duke@435: // Try to coalesce small pages into a large one. iveresov@970: if (UseLargePages && page_size() >= alignment()) { iveresov@970: HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment()); duke@435: if (new_region.contains(p) iveresov@970: && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) { duke@435: if (intersection.contains(p)) { duke@435: intersection = MemRegion(p, intersection.end()); duke@435: } else { duke@435: intersection = MemRegion(p, p); duke@435: } duke@435: } duke@435: } duke@435: *bottom_region = MemRegion(new_region.start(), intersection.start()); duke@435: } else { duke@435: *bottom_region = MemRegion(); duke@435: } duke@435: duke@435: // Is there top? duke@435: if (intersection.end() < new_region.end()) { // Yes duke@435: // Try to coalesce small pages into a large one. iveresov@970: if (UseLargePages && page_size() >= alignment()) { iveresov@970: HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment()); duke@435: if (new_region.contains(p) iveresov@970: && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) { duke@435: if (intersection.contains(p)) { duke@435: intersection = MemRegion(intersection.start(), p); duke@435: } else { duke@435: intersection = MemRegion(p, p); duke@435: } duke@435: } duke@435: } duke@435: *top_region = MemRegion(intersection.end(), new_region.end()); duke@435: } else { duke@435: *top_region = MemRegion(); duke@435: } duke@435: } duke@435: duke@435: // Try to merge the invalid region with the bottom or top region by decreasing duke@435: // the intersection area. Return the invalid_region aligned to the page_size() duke@435: // boundary if it's inside the intersection. Return non-empty invalid_region duke@435: // if it lies inside the intersection (also page-aligned). duke@435: // |------------------new_region---------------------------------| duke@435: // |----------------|-------invalid---|--------------------------| duke@435: // |----bottom_region--|---intersection---|------top_region------| duke@435: void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection, duke@435: MemRegion *invalid_region) { duke@435: if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) { duke@435: *intersection = MemRegion(invalid_region->end(), intersection->end()); duke@435: *invalid_region = MemRegion(); duke@435: } else duke@435: if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) { duke@435: *intersection = MemRegion(intersection->start(), invalid_region->start()); duke@435: *invalid_region = MemRegion(); duke@435: } else duke@435: if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) { duke@435: *intersection = MemRegion(new_region.start(), new_region.start()); duke@435: *invalid_region = MemRegion(); duke@435: } else duke@435: if (intersection->contains(invalid_region)) { duke@435: // That's the only case we have to make an additional bias_region() call. duke@435: HeapWord* start = invalid_region->start(); duke@435: HeapWord* end = invalid_region->end(); iveresov@970: if (UseLargePages && page_size() >= alignment()) { iveresov@970: HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment()); duke@435: if (new_region.contains(p)) { duke@435: start = p; duke@435: } iveresov@970: p = (HeapWord*)round_to((intptr_t) end, alignment()); duke@435: if (new_region.contains(end)) { duke@435: end = p; duke@435: } duke@435: } duke@435: if (intersection->start() > start) { duke@435: *intersection = MemRegion(start, intersection->end()); duke@435: } duke@435: if (intersection->end() < end) { duke@435: *intersection = MemRegion(intersection->start(), end); duke@435: } duke@435: *invalid_region = MemRegion(start, end); duke@435: } duke@435: } duke@435: jmasa@698: void MutableNUMASpace::initialize(MemRegion mr, jmasa@698: bool clear_space, iveresov@970: bool mangle_space, iveresov@970: bool setup_pages) { duke@435: assert(clear_space, "Reallocation will destory data!"); duke@435: assert(lgrp_spaces()->length() > 0, "There should be at least one space"); duke@435: duke@435: MemRegion old_region = region(), new_region; duke@435: set_bottom(mr.start()); duke@435: set_end(mr.end()); jmasa@698: // Must always clear the space jmasa@698: clear(SpaceDecorator::DontMangle); duke@435: duke@435: // Compute chunk sizes duke@435: size_t prev_page_size = page_size(); iveresov@970: set_page_size(UseLargePages ? alignment() : os::vm_page_size()); duke@435: HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); duke@435: HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); duke@435: size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); duke@435: duke@435: // Try small pages if the chunk size is too small duke@435: if (base_space_size_pages / lgrp_spaces()->length() == 0 duke@435: && page_size() > (size_t)os::vm_page_size()) { duke@435: set_page_size(os::vm_page_size()); duke@435: rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); duke@435: rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); duke@435: base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); duke@435: } duke@435: guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small"); duke@435: set_base_space_size(base_space_size_pages); duke@435: duke@435: // Handle space resize duke@435: MemRegion top_region, bottom_region; duke@435: if (!old_region.equals(region())) { duke@435: new_region = MemRegion(rounded_bottom, rounded_end); duke@435: MemRegion intersection = new_region.intersection(old_region); duke@435: if (intersection.start() == NULL || duke@435: intersection.end() == NULL || duke@435: prev_page_size > page_size()) { // If the page size got smaller we have to change duke@435: // the page size preference for the whole space. duke@435: intersection = MemRegion(new_region.start(), new_region.start()); duke@435: } duke@435: select_tails(new_region, intersection, &bottom_region, &top_region); iveresov@576: bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id()); iveresov@576: bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id()); duke@435: } duke@435: duke@435: // Check if the space layout has changed significantly? duke@435: // This happens when the space has been resized so that either head or tail duke@435: // chunk became less than a page. duke@435: bool layout_valid = UseAdaptiveNUMAChunkSizing && duke@435: current_chunk_size(0) > page_size() && duke@435: current_chunk_size(lgrp_spaces()->length() - 1) > page_size(); duke@435: duke@435: duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: LGRPSpace *ls = lgrp_spaces()->at(i); duke@435: MutableSpace *s = ls->space(); duke@435: old_region = s->region(); duke@435: duke@435: size_t chunk_byte_size = 0, old_chunk_byte_size = 0; duke@435: if (i < lgrp_spaces()->length() - 1) { duke@435: if (!UseAdaptiveNUMAChunkSizing || duke@435: (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) || duke@435: samples_count() < AdaptiveSizePolicyReadyThreshold) { duke@435: // No adaptation. Divide the space equally. duke@435: chunk_byte_size = default_chunk_size(); duke@435: } else duke@435: if (!layout_valid || NUMASpaceResizeRate == 0) { duke@435: // Fast adaptation. If no space resize rate is set, resize duke@435: // the chunks instantly. duke@435: chunk_byte_size = adaptive_chunk_size(i, 0); duke@435: } else { duke@435: // Slow adaptation. Resize the chunks moving no more than duke@435: // NUMASpaceResizeRate bytes per collection. duke@435: size_t limit = NUMASpaceResizeRate / duke@435: (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2); duke@435: chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size())); duke@435: } duke@435: duke@435: assert(chunk_byte_size >= page_size(), "Chunk size too small"); duke@435: assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check"); duke@435: } duke@435: duke@435: if (i == 0) { // Bottom chunk duke@435: if (i != lgrp_spaces()->length() - 1) { duke@435: new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize)); duke@435: } else { duke@435: new_region = MemRegion(bottom(), end()); duke@435: } duke@435: } else duke@435: if (i < lgrp_spaces()->length() - 1) { // Middle chunks duke@435: MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); duke@435: new_region = MemRegion(ps->end(), duke@435: ps->end() + (chunk_byte_size >> LogHeapWordSize)); duke@435: } else { // Top chunk duke@435: MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); duke@435: new_region = MemRegion(ps->end(), end()); duke@435: } duke@435: guarantee(region().contains(new_region), "Region invariant"); duke@435: duke@435: duke@435: // The general case: duke@435: // |---------------------|--invalid---|--------------------------| duke@435: // |------------------new_region---------------------------------| duke@435: // |----bottom_region--|---intersection---|------top_region------| duke@435: // |----old_region----| duke@435: // The intersection part has all pages in place we don't need to migrate them. duke@435: // Pages for the top and bottom part should be freed and then reallocated. duke@435: duke@435: MemRegion intersection = old_region.intersection(new_region); duke@435: duke@435: if (intersection.start() == NULL || intersection.end() == NULL) { duke@435: intersection = MemRegion(new_region.start(), new_region.start()); duke@435: } duke@435: iveresov@576: if (!os::numa_has_static_binding()) { iveresov@576: MemRegion invalid_region = ls->invalid_region().intersection(new_region); iveresov@576: // Invalid region is a range of memory that could've possibly iveresov@576: // been allocated on the other node. That's relevant only on Solaris where iveresov@576: // there is no static memory binding. iveresov@576: if (!invalid_region.is_empty()) { iveresov@576: merge_regions(new_region, &intersection, &invalid_region); iveresov@576: free_region(invalid_region); iveresov@576: ls->set_invalid_region(MemRegion()); iveresov@576: } duke@435: } iveresov@576: duke@435: select_tails(new_region, intersection, &bottom_region, &top_region); iveresov@576: iveresov@576: if (!os::numa_has_static_binding()) { iveresov@576: // If that's a system with the first-touch policy then it's enough iveresov@576: // to free the pages. iveresov@576: free_region(bottom_region); iveresov@576: free_region(top_region); iveresov@576: } else { iveresov@576: // In a system with static binding we have to change the bias whenever iveresov@576: // we reshape the heap. iveresov@576: bias_region(bottom_region, ls->lgrp_id()); iveresov@576: bias_region(top_region, ls->lgrp_id()); iveresov@576: } duke@435: jmasa@698: // Clear space (set top = bottom) but never mangle. iveresov@970: s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages); duke@435: duke@435: set_adaptation_cycles(samples_count()); duke@435: } duke@435: } duke@435: duke@435: // Set the top of the whole space. duke@435: // Mark the the holes in chunks below the top() as invalid. duke@435: void MutableNUMASpace::set_top(HeapWord* value) { duke@435: bool found_top = false; iveresov@625: for (int i = 0; i < lgrp_spaces()->length();) { duke@435: LGRPSpace *ls = lgrp_spaces()->at(i); duke@435: MutableSpace *s = ls->space(); duke@435: HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); duke@435: duke@435: if (s->contains(value)) { iveresov@625: // Check if setting the chunk's top to a given value would create a hole less than iveresov@625: // a minimal object; assuming that's not the last chunk in which case we don't care. iveresov@625: if (i < lgrp_spaces()->length() - 1) { iveresov@625: size_t remainder = pointer_delta(s->end(), value); jcoomes@916: const size_t min_fill_size = CollectedHeap::min_fill_size(); jcoomes@916: if (remainder < min_fill_size && remainder > 0) { jcoomes@916: // Add a minimum size filler object; it will cross the chunk boundary. jcoomes@916: CollectedHeap::fill_with_object(value, min_fill_size); jcoomes@916: value += min_fill_size; iveresov@625: assert(!s->contains(value), "Should be in the next chunk"); iveresov@625: // Restart the loop from the same chunk, since the value has moved iveresov@625: // to the next one. iveresov@625: continue; iveresov@625: } iveresov@625: } iveresov@625: iveresov@576: if (!os::numa_has_static_binding() && top < value && top < s->end()) { duke@435: ls->add_invalid_region(MemRegion(top, value)); duke@435: } duke@435: s->set_top(value); duke@435: found_top = true; duke@435: } else { duke@435: if (found_top) { duke@435: s->set_top(s->bottom()); duke@435: } else { iveresov@576: if (!os::numa_has_static_binding() && top < s->end()) { iveresov@576: ls->add_invalid_region(MemRegion(top, s->end())); iveresov@576: } iveresov@576: s->set_top(s->end()); duke@435: } duke@435: } iveresov@625: i++; duke@435: } duke@435: MutableSpace::set_top(value); duke@435: } duke@435: jmasa@698: void MutableNUMASpace::clear(bool mangle_space) { duke@435: MutableSpace::set_top(bottom()); duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { jmasa@698: // Never mangle NUMA spaces because the mangling will jmasa@698: // bind the memory to a possibly unwanted lgroup. jmasa@698: lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle); duke@435: } duke@435: } duke@435: iveresov@576: /* iveresov@576: Linux supports static memory binding, therefore the most part of the iveresov@576: logic dealing with the possible invalid page allocation is effectively iveresov@576: disabled. Besides there is no notion of the home node in Linux. A iveresov@576: thread is allowed to migrate freely. Although the scheduler is rather iveresov@576: reluctant to move threads between the nodes. We check for the current iveresov@576: node every allocation. And with a high probability a thread stays on iveresov@576: the same node for some time allowing local access to recently allocated iveresov@576: objects. iveresov@576: */ iveresov@576: duke@435: HeapWord* MutableNUMASpace::allocate(size_t size) { iveresov@576: Thread* thr = Thread::current(); iveresov@576: int lgrp_id = thr->lgrp_id(); iveresov@576: if (lgrp_id == -1 || !os::numa_has_group_homing()) { duke@435: lgrp_id = os::numa_get_group_id(); iveresov@576: thr->set_lgrp_id(lgrp_id); duke@435: } duke@435: duke@435: int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); duke@435: duke@435: // It is possible that a new CPU has been hotplugged and duke@435: // we haven't reshaped the space accordingly. duke@435: if (i == -1) { duke@435: i = os::random() % lgrp_spaces()->length(); duke@435: } duke@435: iveresov@808: LGRPSpace* ls = lgrp_spaces()->at(i); iveresov@808: MutableSpace *s = ls->space(); duke@435: HeapWord *p = s->allocate(size); duke@435: iveresov@579: if (p != NULL) { iveresov@579: size_t remainder = s->free_in_words(); kvn@1926: if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { iveresov@579: s->set_top(s->top() - size); iveresov@579: p = NULL; iveresov@579: } duke@435: } duke@435: if (p != NULL) { duke@435: if (top() < s->top()) { // Keep _top updated. duke@435: MutableSpace::set_top(s->top()); duke@435: } duke@435: } iveresov@576: // Make the page allocation happen here if there is no static binding.. iveresov@576: if (p != NULL && !os::numa_has_static_binding()) { duke@435: for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { duke@435: *(int*)i = 0; duke@435: } duke@435: } iveresov@808: if (p == NULL) { iveresov@808: ls->set_allocation_failed(); iveresov@808: } duke@435: return p; duke@435: } duke@435: duke@435: // This version is lock-free. duke@435: HeapWord* MutableNUMASpace::cas_allocate(size_t size) { iveresov@576: Thread* thr = Thread::current(); iveresov@576: int lgrp_id = thr->lgrp_id(); iveresov@576: if (lgrp_id == -1 || !os::numa_has_group_homing()) { duke@435: lgrp_id = os::numa_get_group_id(); iveresov@576: thr->set_lgrp_id(lgrp_id); duke@435: } duke@435: duke@435: int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); duke@435: // It is possible that a new CPU has been hotplugged and duke@435: // we haven't reshaped the space accordingly. duke@435: if (i == -1) { duke@435: i = os::random() % lgrp_spaces()->length(); duke@435: } iveresov@808: LGRPSpace *ls = lgrp_spaces()->at(i); iveresov@808: MutableSpace *s = ls->space(); duke@435: HeapWord *p = s->cas_allocate(size); iveresov@579: if (p != NULL) { iveresov@625: size_t remainder = pointer_delta(s->end(), p + size); kvn@1926: if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { iveresov@579: if (s->cas_deallocate(p, size)) { iveresov@579: // We were the last to allocate and created a fragment less than iveresov@579: // a minimal object. iveresov@579: p = NULL; iveresov@625: } else { iveresov@625: guarantee(false, "Deallocation should always succeed"); iveresov@579: } duke@435: } duke@435: } duke@435: if (p != NULL) { duke@435: HeapWord* cur_top, *cur_chunk_top = p + size; duke@435: while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated. duke@435: if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) { duke@435: break; duke@435: } duke@435: } duke@435: } duke@435: iveresov@576: // Make the page allocation happen here if there is no static binding. iveresov@576: if (p != NULL && !os::numa_has_static_binding() ) { duke@435: for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { duke@435: *(int*)i = 0; duke@435: } duke@435: } iveresov@808: if (p == NULL) { iveresov@808: ls->set_allocation_failed(); iveresov@808: } duke@435: return p; duke@435: } duke@435: duke@435: void MutableNUMASpace::print_short_on(outputStream* st) const { duke@435: MutableSpace::print_short_on(st); duke@435: st->print(" ("); duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id()); duke@435: lgrp_spaces()->at(i)->space()->print_short_on(st); duke@435: if (i < lgrp_spaces()->length() - 1) { duke@435: st->print(", "); duke@435: } duke@435: } duke@435: st->print(")"); duke@435: } duke@435: duke@435: void MutableNUMASpace::print_on(outputStream* st) const { duke@435: MutableSpace::print_on(st); duke@435: for (int i = 0; i < lgrp_spaces()->length(); i++) { duke@435: LGRPSpace *ls = lgrp_spaces()->at(i); duke@435: st->print(" lgrp %d", ls->lgrp_id()); duke@435: ls->space()->print_on(st); duke@435: if (NUMAStats) { iveresov@579: for (int i = 0; i < lgrp_spaces()->length(); i++) { iveresov@579: lgrp_spaces()->at(i)->accumulate_statistics(page_size()); iveresov@579: } duke@435: st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n", duke@435: ls->space_stats()->_local_space / K, duke@435: ls->space_stats()->_remote_space / K, duke@435: ls->space_stats()->_unbiased_space / K, duke@435: ls->space_stats()->_uncommited_space / K, duke@435: ls->space_stats()->_large_pages, duke@435: ls->space_stats()->_small_pages); duke@435: } duke@435: } duke@435: } duke@435: brutisso@3711: void MutableNUMASpace::verify() { iveresov@625: // This can be called after setting an arbitary value to the space's top, iveresov@625: // so an object can cross the chunk boundary. We ensure the parsablity iveresov@625: // of the space and just walk the objects in linear fashion. iveresov@625: ensure_parsability(); brutisso@3711: MutableSpace::verify(); duke@435: } duke@435: duke@435: // Scan pages and gather stats about page placement and size. duke@435: void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) { duke@435: clear_space_stats(); duke@435: char *start = (char*)round_to((intptr_t) space()->bottom(), page_size); duke@435: char* end = (char*)round_down((intptr_t) space()->end(), page_size); duke@435: if (start < end) { duke@435: for (char *p = start; p < end;) { duke@435: os::page_info info; duke@435: if (os::get_page_info(p, &info)) { duke@435: if (info.size > 0) { duke@435: if (info.size > (size_t)os::vm_page_size()) { duke@435: space_stats()->_large_pages++; duke@435: } else { duke@435: space_stats()->_small_pages++; duke@435: } duke@435: if (info.lgrp_id == lgrp_id()) { duke@435: space_stats()->_local_space += info.size; duke@435: } else { duke@435: space_stats()->_remote_space += info.size; duke@435: } duke@435: p += info.size; duke@435: } else { duke@435: p += os::vm_page_size(); duke@435: space_stats()->_uncommited_space += os::vm_page_size(); duke@435: } duke@435: } else { duke@435: return; duke@435: } duke@435: } duke@435: } duke@435: space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) + duke@435: pointer_delta(space()->end(), end, sizeof(char)); duke@435: duke@435: } duke@435: duke@435: // Scan page_count pages and verify if they have the right size and right placement. duke@435: // If invalid pages are found they are freed in hope that subsequent reallocation duke@435: // will be more successful. duke@435: void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count) duke@435: { duke@435: char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size); duke@435: char* range_end = (char*)round_down((intptr_t) space()->end(), page_size); duke@435: duke@435: if (range_start > last_page_scanned() || last_page_scanned() >= range_end) { duke@435: set_last_page_scanned(range_start); duke@435: } duke@435: duke@435: char *scan_start = last_page_scanned(); duke@435: char* scan_end = MIN2(scan_start + page_size * page_count, range_end); duke@435: duke@435: os::page_info page_expected, page_found; duke@435: page_expected.size = page_size; duke@435: page_expected.lgrp_id = lgrp_id(); duke@435: duke@435: char *s = scan_start; duke@435: while (s < scan_end) { duke@435: char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found); duke@435: if (e == NULL) { duke@435: break; duke@435: } duke@435: if (e != scan_end) { stefank@4739: assert(e < scan_end, err_msg("e: " PTR_FORMAT " scan_end: " PTR_FORMAT, e, scan_end)); stefank@4739: duke@435: if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id()) duke@435: && page_expected.size != 0) { iveresov@3363: os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size); duke@435: } duke@435: page_expected = page_found; duke@435: } duke@435: s = e; duke@435: } duke@435: duke@435: set_last_page_scanned(scan_end); duke@435: }