Mon, 27 Jan 2014 13:14:53 +0100
8030177: G1: Enable TLAB resizing
Reviewed-by: tschatzl, stefank, jmasa
2 /*
3 * Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
26 #include "precompiled.hpp"
27 #include "gc_implementation/shared/mutableNUMASpace.hpp"
28 #include "gc_implementation/shared/spaceDecorator.hpp"
29 #include "memory/sharedHeap.hpp"
30 #include "oops/oop.inline.hpp"
31 #include "runtime/thread.inline.hpp"
33 MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) {
34 _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray<LGRPSpace*>(0, true);
35 _page_size = os::vm_page_size();
36 _adaptation_cycles = 0;
37 _samples_count = 0;
38 update_layout(true);
39 }
41 MutableNUMASpace::~MutableNUMASpace() {
42 for (int i = 0; i < lgrp_spaces()->length(); i++) {
43 delete lgrp_spaces()->at(i);
44 }
45 delete lgrp_spaces();
46 }
48 #ifndef PRODUCT
49 void MutableNUMASpace::mangle_unused_area() {
50 // This method should do nothing.
51 // It can be called on a numa space during a full compaction.
52 }
53 void MutableNUMASpace::mangle_unused_area_complete() {
54 // This method should do nothing.
55 // It can be called on a numa space during a full compaction.
56 }
57 void MutableNUMASpace::mangle_region(MemRegion mr) {
58 // This method should do nothing because numa spaces are not mangled.
59 }
60 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
61 assert(false, "Do not mangle MutableNUMASpace's");
62 }
63 void MutableNUMASpace::set_top_for_allocations() {
64 // This method should do nothing.
65 }
66 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
67 // This method should do nothing.
68 }
69 void MutableNUMASpace::check_mangled_unused_area_complete() {
70 // This method should do nothing.
71 }
72 #endif // NOT_PRODUCT
74 // There may be unallocated holes in the middle chunks
75 // that should be filled with dead objects to ensure parseability.
76 void MutableNUMASpace::ensure_parsability() {
77 for (int i = 0; i < lgrp_spaces()->length(); i++) {
78 LGRPSpace *ls = lgrp_spaces()->at(i);
79 MutableSpace *s = ls->space();
80 if (s->top() < top()) { // For all spaces preceding the one containing top()
81 if (s->free_in_words() > 0) {
82 intptr_t cur_top = (intptr_t)s->top();
83 size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
84 while (words_left_to_fill > 0) {
85 size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
86 assert(words_to_fill >= CollectedHeap::min_fill_size(),
87 err_msg("Remaining size ("SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
88 words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size()));
89 CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
90 if (!os::numa_has_static_binding()) {
91 size_t touched_words = words_to_fill;
92 #ifndef ASSERT
93 if (!ZapUnusedHeapArea) {
94 touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
95 touched_words);
96 }
97 #endif
98 MemRegion invalid;
99 HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
100 HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
101 if (crossing_start != crossing_end) {
102 // If object header crossed a small page boundary we mark the area
103 // as invalid rounding it to a page_size().
104 HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
105 HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
106 invalid = MemRegion(start, end);
107 }
109 ls->add_invalid_region(invalid);
110 }
111 cur_top = cur_top + (words_to_fill * HeapWordSize);
112 words_left_to_fill -= words_to_fill;
113 }
114 }
115 } else {
116 if (!os::numa_has_static_binding()) {
117 #ifdef ASSERT
118 MemRegion invalid(s->top(), s->end());
119 ls->add_invalid_region(invalid);
120 #else
121 if (ZapUnusedHeapArea) {
122 MemRegion invalid(s->top(), s->end());
123 ls->add_invalid_region(invalid);
124 } else {
125 return;
126 }
127 #endif
128 } else {
129 return;
130 }
131 }
132 }
133 }
135 size_t MutableNUMASpace::used_in_words() const {
136 size_t s = 0;
137 for (int i = 0; i < lgrp_spaces()->length(); i++) {
138 s += lgrp_spaces()->at(i)->space()->used_in_words();
139 }
140 return s;
141 }
143 size_t MutableNUMASpace::free_in_words() const {
144 size_t s = 0;
145 for (int i = 0; i < lgrp_spaces()->length(); i++) {
146 s += lgrp_spaces()->at(i)->space()->free_in_words();
147 }
148 return s;
149 }
152 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
153 guarantee(thr != NULL, "No thread");
154 int lgrp_id = thr->lgrp_id();
155 if (lgrp_id == -1) {
156 // This case can occur after the topology of the system has
157 // changed. Thread can change their location, the new home
158 // group will be determined during the first allocation
159 // attempt. For now we can safely assume that all spaces
160 // have equal size because the whole space will be reinitialized.
161 if (lgrp_spaces()->length() > 0) {
162 return capacity_in_bytes() / lgrp_spaces()->length();
163 } else {
164 assert(false, "There should be at least one locality group");
165 return 0;
166 }
167 }
168 // That's the normal case, where we know the locality group of the thread.
169 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
170 if (i == -1) {
171 return 0;
172 }
173 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
174 }
176 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
177 // Please see the comments for tlab_capacity().
178 guarantee(thr != NULL, "No thread");
179 int lgrp_id = thr->lgrp_id();
180 if (lgrp_id == -1) {
181 if (lgrp_spaces()->length() > 0) {
182 return (used_in_bytes()) / lgrp_spaces()->length();
183 } else {
184 assert(false, "There should be at least one locality group");
185 return 0;
186 }
187 }
188 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
189 if (i == -1) {
190 return 0;
191 }
192 return lgrp_spaces()->at(i)->space()->used_in_bytes();
193 }
196 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
197 // Please see the comments for tlab_capacity().
198 guarantee(thr != NULL, "No thread");
199 int lgrp_id = thr->lgrp_id();
200 if (lgrp_id == -1) {
201 if (lgrp_spaces()->length() > 0) {
202 return free_in_bytes() / lgrp_spaces()->length();
203 } else {
204 assert(false, "There should be at least one locality group");
205 return 0;
206 }
207 }
208 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
209 if (i == -1) {
210 return 0;
211 }
212 return lgrp_spaces()->at(i)->space()->free_in_bytes();
213 }
216 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
217 guarantee(thr != NULL, "No thread");
218 int lgrp_id = thr->lgrp_id();
219 if (lgrp_id == -1) {
220 if (lgrp_spaces()->length() > 0) {
221 return capacity_in_words() / lgrp_spaces()->length();
222 } else {
223 assert(false, "There should be at least one locality group");
224 return 0;
225 }
226 }
227 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
228 if (i == -1) {
229 return 0;
230 }
231 return lgrp_spaces()->at(i)->space()->capacity_in_words();
232 }
234 // Check if the NUMA topology has changed. Add and remove spaces if needed.
235 // The update can be forced by setting the force parameter equal to true.
236 bool MutableNUMASpace::update_layout(bool force) {
237 // Check if the topology had changed.
238 bool changed = os::numa_topology_changed();
239 if (force || changed) {
240 // Compute lgrp intersection. Add/remove spaces.
241 int lgrp_limit = (int)os::numa_get_groups_num();
242 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC);
243 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
244 assert(lgrp_num > 0, "There should be at least one locality group");
245 // Add new spaces for the new nodes
246 for (int i = 0; i < lgrp_num; i++) {
247 bool found = false;
248 for (int j = 0; j < lgrp_spaces()->length(); j++) {
249 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
250 found = true;
251 break;
252 }
253 }
254 if (!found) {
255 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment()));
256 }
257 }
259 // Remove spaces for the removed nodes.
260 for (int i = 0; i < lgrp_spaces()->length();) {
261 bool found = false;
262 for (int j = 0; j < lgrp_num; j++) {
263 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
264 found = true;
265 break;
266 }
267 }
268 if (!found) {
269 delete lgrp_spaces()->at(i);
270 lgrp_spaces()->remove_at(i);
271 } else {
272 i++;
273 }
274 }
276 FREE_C_HEAP_ARRAY(int, lgrp_ids, mtGC);
278 if (changed) {
279 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
280 thread->set_lgrp_id(-1);
281 }
282 }
283 return true;
284 }
285 return false;
286 }
288 // Bias region towards the first-touching lgrp. Set the right page sizes.
289 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
290 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
291 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
292 if (end > start) {
293 MemRegion aligned_region(start, end);
294 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
295 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
296 assert(region().contains(aligned_region), "Sanity");
297 // First we tell the OS which page size we want in the given range. The underlying
298 // large page can be broken down if we require small pages.
299 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
300 // Then we uncommit the pages in the range.
301 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
302 // And make them local/first-touch biased.
303 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
304 }
305 }
307 // Free all pages in the region.
308 void MutableNUMASpace::free_region(MemRegion mr) {
309 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
310 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
311 if (end > start) {
312 MemRegion aligned_region(start, end);
313 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
314 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
315 assert(region().contains(aligned_region), "Sanity");
316 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
317 }
318 }
320 // Update space layout. Perform adaptation.
321 void MutableNUMASpace::update() {
322 if (update_layout(false)) {
323 // If the topology has changed, make all chunks zero-sized.
324 // And clear the alloc-rate statistics.
325 // In future we may want to handle this more gracefully in order
326 // to avoid the reallocation of the pages as much as possible.
327 for (int i = 0; i < lgrp_spaces()->length(); i++) {
328 LGRPSpace *ls = lgrp_spaces()->at(i);
329 MutableSpace *s = ls->space();
330 s->set_end(s->bottom());
331 s->set_top(s->bottom());
332 ls->clear_alloc_rate();
333 }
334 // A NUMA space is never mangled
335 initialize(region(),
336 SpaceDecorator::Clear,
337 SpaceDecorator::DontMangle);
338 } else {
339 bool should_initialize = false;
340 if (!os::numa_has_static_binding()) {
341 for (int i = 0; i < lgrp_spaces()->length(); i++) {
342 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
343 should_initialize = true;
344 break;
345 }
346 }
347 }
349 if (should_initialize ||
350 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
351 // A NUMA space is never mangled
352 initialize(region(),
353 SpaceDecorator::Clear,
354 SpaceDecorator::DontMangle);
355 }
356 }
358 if (NUMAStats) {
359 for (int i = 0; i < lgrp_spaces()->length(); i++) {
360 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
361 }
362 }
364 scan_pages(NUMAPageScanRate);
365 }
367 // Scan pages. Free pages that have smaller size or wrong placement.
368 void MutableNUMASpace::scan_pages(size_t page_count)
369 {
370 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
371 if (pages_per_chunk > 0) {
372 for (int i = 0; i < lgrp_spaces()->length(); i++) {
373 LGRPSpace *ls = lgrp_spaces()->at(i);
374 ls->scan_pages(page_size(), pages_per_chunk);
375 }
376 }
377 }
379 // Accumulate statistics about the allocation rate of each lgrp.
380 void MutableNUMASpace::accumulate_statistics() {
381 if (UseAdaptiveNUMAChunkSizing) {
382 for (int i = 0; i < lgrp_spaces()->length(); i++) {
383 lgrp_spaces()->at(i)->sample();
384 }
385 increment_samples_count();
386 }
388 if (NUMAStats) {
389 for (int i = 0; i < lgrp_spaces()->length(); i++) {
390 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
391 }
392 }
393 }
395 // Get the current size of a chunk.
396 // This function computes the size of the chunk based on the
397 // difference between chunk ends. This allows it to work correctly in
398 // case the whole space is resized and during the process of adaptive
399 // chunk resizing.
400 size_t MutableNUMASpace::current_chunk_size(int i) {
401 HeapWord *cur_end, *prev_end;
402 if (i == 0) {
403 prev_end = bottom();
404 } else {
405 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
406 }
407 if (i == lgrp_spaces()->length() - 1) {
408 cur_end = end();
409 } else {
410 cur_end = lgrp_spaces()->at(i)->space()->end();
411 }
412 if (cur_end > prev_end) {
413 return pointer_delta(cur_end, prev_end, sizeof(char));
414 }
415 return 0;
416 }
418 // Return the default chunk size by equally diving the space.
419 // page_size() aligned.
420 size_t MutableNUMASpace::default_chunk_size() {
421 return base_space_size() / lgrp_spaces()->length() * page_size();
422 }
424 // Produce a new chunk size. page_size() aligned.
425 // This function is expected to be called on sequence of i's from 0 to
426 // lgrp_spaces()->length().
427 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
428 size_t pages_available = base_space_size();
429 for (int j = 0; j < i; j++) {
430 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
431 }
432 pages_available -= lgrp_spaces()->length() - i - 1;
433 assert(pages_available > 0, "No pages left");
434 float alloc_rate = 0;
435 for (int j = i; j < lgrp_spaces()->length(); j++) {
436 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
437 }
438 size_t chunk_size = 0;
439 if (alloc_rate > 0) {
440 LGRPSpace *ls = lgrp_spaces()->at(i);
441 chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size();
442 }
443 chunk_size = MAX2(chunk_size, page_size());
445 if (limit > 0) {
446 limit = round_down(limit, page_size());
447 if (chunk_size > current_chunk_size(i)) {
448 size_t upper_bound = pages_available * page_size();
449 if (upper_bound > limit &&
450 current_chunk_size(i) < upper_bound - limit) {
451 // The resulting upper bound should not exceed the available
452 // amount of memory (pages_available * page_size()).
453 upper_bound = current_chunk_size(i) + limit;
454 }
455 chunk_size = MIN2(chunk_size, upper_bound);
456 } else {
457 size_t lower_bound = page_size();
458 if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow.
459 lower_bound = current_chunk_size(i) - limit;
460 }
461 chunk_size = MAX2(chunk_size, lower_bound);
462 }
463 }
464 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
465 return chunk_size;
466 }
469 // Return the bottom_region and the top_region. Align them to page_size() boundary.
470 // |------------------new_region---------------------------------|
471 // |----bottom_region--|---intersection---|------top_region------|
472 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
473 MemRegion* bottom_region, MemRegion *top_region) {
474 // Is there bottom?
475 if (new_region.start() < intersection.start()) { // Yes
476 // Try to coalesce small pages into a large one.
477 if (UseLargePages && page_size() >= alignment()) {
478 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment());
479 if (new_region.contains(p)
480 && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
481 if (intersection.contains(p)) {
482 intersection = MemRegion(p, intersection.end());
483 } else {
484 intersection = MemRegion(p, p);
485 }
486 }
487 }
488 *bottom_region = MemRegion(new_region.start(), intersection.start());
489 } else {
490 *bottom_region = MemRegion();
491 }
493 // Is there top?
494 if (intersection.end() < new_region.end()) { // Yes
495 // Try to coalesce small pages into a large one.
496 if (UseLargePages && page_size() >= alignment()) {
497 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment());
498 if (new_region.contains(p)
499 && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
500 if (intersection.contains(p)) {
501 intersection = MemRegion(intersection.start(), p);
502 } else {
503 intersection = MemRegion(p, p);
504 }
505 }
506 }
507 *top_region = MemRegion(intersection.end(), new_region.end());
508 } else {
509 *top_region = MemRegion();
510 }
511 }
513 // Try to merge the invalid region with the bottom or top region by decreasing
514 // the intersection area. Return the invalid_region aligned to the page_size()
515 // boundary if it's inside the intersection. Return non-empty invalid_region
516 // if it lies inside the intersection (also page-aligned).
517 // |------------------new_region---------------------------------|
518 // |----------------|-------invalid---|--------------------------|
519 // |----bottom_region--|---intersection---|------top_region------|
520 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
521 MemRegion *invalid_region) {
522 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
523 *intersection = MemRegion(invalid_region->end(), intersection->end());
524 *invalid_region = MemRegion();
525 } else
526 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
527 *intersection = MemRegion(intersection->start(), invalid_region->start());
528 *invalid_region = MemRegion();
529 } else
530 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
531 *intersection = MemRegion(new_region.start(), new_region.start());
532 *invalid_region = MemRegion();
533 } else
534 if (intersection->contains(invalid_region)) {
535 // That's the only case we have to make an additional bias_region() call.
536 HeapWord* start = invalid_region->start();
537 HeapWord* end = invalid_region->end();
538 if (UseLargePages && page_size() >= alignment()) {
539 HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment());
540 if (new_region.contains(p)) {
541 start = p;
542 }
543 p = (HeapWord*)round_to((intptr_t) end, alignment());
544 if (new_region.contains(end)) {
545 end = p;
546 }
547 }
548 if (intersection->start() > start) {
549 *intersection = MemRegion(start, intersection->end());
550 }
551 if (intersection->end() < end) {
552 *intersection = MemRegion(intersection->start(), end);
553 }
554 *invalid_region = MemRegion(start, end);
555 }
556 }
558 void MutableNUMASpace::initialize(MemRegion mr,
559 bool clear_space,
560 bool mangle_space,
561 bool setup_pages) {
562 assert(clear_space, "Reallocation will destory data!");
563 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
565 MemRegion old_region = region(), new_region;
566 set_bottom(mr.start());
567 set_end(mr.end());
568 // Must always clear the space
569 clear(SpaceDecorator::DontMangle);
571 // Compute chunk sizes
572 size_t prev_page_size = page_size();
573 set_page_size(UseLargePages ? alignment() : os::vm_page_size());
574 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
575 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
576 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
578 // Try small pages if the chunk size is too small
579 if (base_space_size_pages / lgrp_spaces()->length() == 0
580 && page_size() > (size_t)os::vm_page_size()) {
581 set_page_size(os::vm_page_size());
582 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
583 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
584 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
585 }
586 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
587 set_base_space_size(base_space_size_pages);
589 // Handle space resize
590 MemRegion top_region, bottom_region;
591 if (!old_region.equals(region())) {
592 new_region = MemRegion(rounded_bottom, rounded_end);
593 MemRegion intersection = new_region.intersection(old_region);
594 if (intersection.start() == NULL ||
595 intersection.end() == NULL ||
596 prev_page_size > page_size()) { // If the page size got smaller we have to change
597 // the page size preference for the whole space.
598 intersection = MemRegion(new_region.start(), new_region.start());
599 }
600 select_tails(new_region, intersection, &bottom_region, &top_region);
601 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
602 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
603 }
605 // Check if the space layout has changed significantly?
606 // This happens when the space has been resized so that either head or tail
607 // chunk became less than a page.
608 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
609 current_chunk_size(0) > page_size() &&
610 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
613 for (int i = 0; i < lgrp_spaces()->length(); i++) {
614 LGRPSpace *ls = lgrp_spaces()->at(i);
615 MutableSpace *s = ls->space();
616 old_region = s->region();
618 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
619 if (i < lgrp_spaces()->length() - 1) {
620 if (!UseAdaptiveNUMAChunkSizing ||
621 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
622 samples_count() < AdaptiveSizePolicyReadyThreshold) {
623 // No adaptation. Divide the space equally.
624 chunk_byte_size = default_chunk_size();
625 } else
626 if (!layout_valid || NUMASpaceResizeRate == 0) {
627 // Fast adaptation. If no space resize rate is set, resize
628 // the chunks instantly.
629 chunk_byte_size = adaptive_chunk_size(i, 0);
630 } else {
631 // Slow adaptation. Resize the chunks moving no more than
632 // NUMASpaceResizeRate bytes per collection.
633 size_t limit = NUMASpaceResizeRate /
634 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
635 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
636 }
638 assert(chunk_byte_size >= page_size(), "Chunk size too small");
639 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
640 }
642 if (i == 0) { // Bottom chunk
643 if (i != lgrp_spaces()->length() - 1) {
644 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
645 } else {
646 new_region = MemRegion(bottom(), end());
647 }
648 } else
649 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
650 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
651 new_region = MemRegion(ps->end(),
652 ps->end() + (chunk_byte_size >> LogHeapWordSize));
653 } else { // Top chunk
654 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
655 new_region = MemRegion(ps->end(), end());
656 }
657 guarantee(region().contains(new_region), "Region invariant");
660 // The general case:
661 // |---------------------|--invalid---|--------------------------|
662 // |------------------new_region---------------------------------|
663 // |----bottom_region--|---intersection---|------top_region------|
664 // |----old_region----|
665 // The intersection part has all pages in place we don't need to migrate them.
666 // Pages for the top and bottom part should be freed and then reallocated.
668 MemRegion intersection = old_region.intersection(new_region);
670 if (intersection.start() == NULL || intersection.end() == NULL) {
671 intersection = MemRegion(new_region.start(), new_region.start());
672 }
674 if (!os::numa_has_static_binding()) {
675 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
676 // Invalid region is a range of memory that could've possibly
677 // been allocated on the other node. That's relevant only on Solaris where
678 // there is no static memory binding.
679 if (!invalid_region.is_empty()) {
680 merge_regions(new_region, &intersection, &invalid_region);
681 free_region(invalid_region);
682 ls->set_invalid_region(MemRegion());
683 }
684 }
686 select_tails(new_region, intersection, &bottom_region, &top_region);
688 if (!os::numa_has_static_binding()) {
689 // If that's a system with the first-touch policy then it's enough
690 // to free the pages.
691 free_region(bottom_region);
692 free_region(top_region);
693 } else {
694 // In a system with static binding we have to change the bias whenever
695 // we reshape the heap.
696 bias_region(bottom_region, ls->lgrp_id());
697 bias_region(top_region, ls->lgrp_id());
698 }
700 // Clear space (set top = bottom) but never mangle.
701 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
703 set_adaptation_cycles(samples_count());
704 }
705 }
707 // Set the top of the whole space.
708 // Mark the the holes in chunks below the top() as invalid.
709 void MutableNUMASpace::set_top(HeapWord* value) {
710 bool found_top = false;
711 for (int i = 0; i < lgrp_spaces()->length();) {
712 LGRPSpace *ls = lgrp_spaces()->at(i);
713 MutableSpace *s = ls->space();
714 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
716 if (s->contains(value)) {
717 // Check if setting the chunk's top to a given value would create a hole less than
718 // a minimal object; assuming that's not the last chunk in which case we don't care.
719 if (i < lgrp_spaces()->length() - 1) {
720 size_t remainder = pointer_delta(s->end(), value);
721 const size_t min_fill_size = CollectedHeap::min_fill_size();
722 if (remainder < min_fill_size && remainder > 0) {
723 // Add a minimum size filler object; it will cross the chunk boundary.
724 CollectedHeap::fill_with_object(value, min_fill_size);
725 value += min_fill_size;
726 assert(!s->contains(value), "Should be in the next chunk");
727 // Restart the loop from the same chunk, since the value has moved
728 // to the next one.
729 continue;
730 }
731 }
733 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
734 ls->add_invalid_region(MemRegion(top, value));
735 }
736 s->set_top(value);
737 found_top = true;
738 } else {
739 if (found_top) {
740 s->set_top(s->bottom());
741 } else {
742 if (!os::numa_has_static_binding() && top < s->end()) {
743 ls->add_invalid_region(MemRegion(top, s->end()));
744 }
745 s->set_top(s->end());
746 }
747 }
748 i++;
749 }
750 MutableSpace::set_top(value);
751 }
753 void MutableNUMASpace::clear(bool mangle_space) {
754 MutableSpace::set_top(bottom());
755 for (int i = 0; i < lgrp_spaces()->length(); i++) {
756 // Never mangle NUMA spaces because the mangling will
757 // bind the memory to a possibly unwanted lgroup.
758 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
759 }
760 }
762 /*
763 Linux supports static memory binding, therefore the most part of the
764 logic dealing with the possible invalid page allocation is effectively
765 disabled. Besides there is no notion of the home node in Linux. A
766 thread is allowed to migrate freely. Although the scheduler is rather
767 reluctant to move threads between the nodes. We check for the current
768 node every allocation. And with a high probability a thread stays on
769 the same node for some time allowing local access to recently allocated
770 objects.
771 */
773 HeapWord* MutableNUMASpace::allocate(size_t size) {
774 Thread* thr = Thread::current();
775 int lgrp_id = thr->lgrp_id();
776 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
777 lgrp_id = os::numa_get_group_id();
778 thr->set_lgrp_id(lgrp_id);
779 }
781 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
783 // It is possible that a new CPU has been hotplugged and
784 // we haven't reshaped the space accordingly.
785 if (i == -1) {
786 i = os::random() % lgrp_spaces()->length();
787 }
789 LGRPSpace* ls = lgrp_spaces()->at(i);
790 MutableSpace *s = ls->space();
791 HeapWord *p = s->allocate(size);
793 if (p != NULL) {
794 size_t remainder = s->free_in_words();
795 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
796 s->set_top(s->top() - size);
797 p = NULL;
798 }
799 }
800 if (p != NULL) {
801 if (top() < s->top()) { // Keep _top updated.
802 MutableSpace::set_top(s->top());
803 }
804 }
805 // Make the page allocation happen here if there is no static binding..
806 if (p != NULL && !os::numa_has_static_binding()) {
807 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
808 *(int*)i = 0;
809 }
810 }
811 if (p == NULL) {
812 ls->set_allocation_failed();
813 }
814 return p;
815 }
817 // This version is lock-free.
818 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
819 Thread* thr = Thread::current();
820 int lgrp_id = thr->lgrp_id();
821 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
822 lgrp_id = os::numa_get_group_id();
823 thr->set_lgrp_id(lgrp_id);
824 }
826 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
827 // It is possible that a new CPU has been hotplugged and
828 // we haven't reshaped the space accordingly.
829 if (i == -1) {
830 i = os::random() % lgrp_spaces()->length();
831 }
832 LGRPSpace *ls = lgrp_spaces()->at(i);
833 MutableSpace *s = ls->space();
834 HeapWord *p = s->cas_allocate(size);
835 if (p != NULL) {
836 size_t remainder = pointer_delta(s->end(), p + size);
837 if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
838 if (s->cas_deallocate(p, size)) {
839 // We were the last to allocate and created a fragment less than
840 // a minimal object.
841 p = NULL;
842 } else {
843 guarantee(false, "Deallocation should always succeed");
844 }
845 }
846 }
847 if (p != NULL) {
848 HeapWord* cur_top, *cur_chunk_top = p + size;
849 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
850 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
851 break;
852 }
853 }
854 }
856 // Make the page allocation happen here if there is no static binding.
857 if (p != NULL && !os::numa_has_static_binding() ) {
858 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
859 *(int*)i = 0;
860 }
861 }
862 if (p == NULL) {
863 ls->set_allocation_failed();
864 }
865 return p;
866 }
868 void MutableNUMASpace::print_short_on(outputStream* st) const {
869 MutableSpace::print_short_on(st);
870 st->print(" (");
871 for (int i = 0; i < lgrp_spaces()->length(); i++) {
872 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
873 lgrp_spaces()->at(i)->space()->print_short_on(st);
874 if (i < lgrp_spaces()->length() - 1) {
875 st->print(", ");
876 }
877 }
878 st->print(")");
879 }
881 void MutableNUMASpace::print_on(outputStream* st) const {
882 MutableSpace::print_on(st);
883 for (int i = 0; i < lgrp_spaces()->length(); i++) {
884 LGRPSpace *ls = lgrp_spaces()->at(i);
885 st->print(" lgrp %d", ls->lgrp_id());
886 ls->space()->print_on(st);
887 if (NUMAStats) {
888 for (int i = 0; i < lgrp_spaces()->length(); i++) {
889 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
890 }
891 st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n",
892 ls->space_stats()->_local_space / K,
893 ls->space_stats()->_remote_space / K,
894 ls->space_stats()->_unbiased_space / K,
895 ls->space_stats()->_uncommited_space / K,
896 ls->space_stats()->_large_pages,
897 ls->space_stats()->_small_pages);
898 }
899 }
900 }
902 void MutableNUMASpace::verify() {
903 // This can be called after setting an arbitary value to the space's top,
904 // so an object can cross the chunk boundary. We ensure the parsablity
905 // of the space and just walk the objects in linear fashion.
906 ensure_parsability();
907 MutableSpace::verify();
908 }
910 // Scan pages and gather stats about page placement and size.
911 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
912 clear_space_stats();
913 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
914 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
915 if (start < end) {
916 for (char *p = start; p < end;) {
917 os::page_info info;
918 if (os::get_page_info(p, &info)) {
919 if (info.size > 0) {
920 if (info.size > (size_t)os::vm_page_size()) {
921 space_stats()->_large_pages++;
922 } else {
923 space_stats()->_small_pages++;
924 }
925 if (info.lgrp_id == lgrp_id()) {
926 space_stats()->_local_space += info.size;
927 } else {
928 space_stats()->_remote_space += info.size;
929 }
930 p += info.size;
931 } else {
932 p += os::vm_page_size();
933 space_stats()->_uncommited_space += os::vm_page_size();
934 }
935 } else {
936 return;
937 }
938 }
939 }
940 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
941 pointer_delta(space()->end(), end, sizeof(char));
943 }
945 // Scan page_count pages and verify if they have the right size and right placement.
946 // If invalid pages are found they are freed in hope that subsequent reallocation
947 // will be more successful.
948 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
949 {
950 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
951 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
953 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
954 set_last_page_scanned(range_start);
955 }
957 char *scan_start = last_page_scanned();
958 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
960 os::page_info page_expected, page_found;
961 page_expected.size = page_size;
962 page_expected.lgrp_id = lgrp_id();
964 char *s = scan_start;
965 while (s < scan_end) {
966 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
967 if (e == NULL) {
968 break;
969 }
970 if (e != scan_end) {
971 assert(e < scan_end, err_msg("e: " PTR_FORMAT " scan_end: " PTR_FORMAT, e, scan_end));
973 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
974 && page_expected.size != 0) {
975 os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size);
976 }
977 page_expected = page_found;
978 }
979 s = e;
980 }
982 set_last_page_scanned(scan_end);
983 }