Sat, 27 Sep 2008 00:33:13 -0700
6740923: NUMA allocator: Ensure the progress of adaptive chunk resizing
Summary: Treat a chuck where the allocation has failed as fully used.
Reviewed-by: ysr
2 /*
3 * Copyright 2006-2008 Sun Microsystems, Inc. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
22 * have any questions.
23 *
24 */
26 # include "incls/_precompiled.incl"
27 # include "incls/_mutableNUMASpace.cpp.incl"
30 MutableNUMASpace::MutableNUMASpace() {
31 _lgrp_spaces = new (ResourceObj::C_HEAP) GrowableArray<LGRPSpace*>(0, true);
32 _page_size = os::vm_page_size();
33 _adaptation_cycles = 0;
34 _samples_count = 0;
35 update_layout(true);
36 }
38 MutableNUMASpace::~MutableNUMASpace() {
39 for (int i = 0; i < lgrp_spaces()->length(); i++) {
40 delete lgrp_spaces()->at(i);
41 }
42 delete lgrp_spaces();
43 }
45 #ifndef PRODUCT
46 void MutableNUMASpace::mangle_unused_area() {
47 // This method should do nothing.
48 // It can be called on a numa space during a full compaction.
49 }
50 void MutableNUMASpace::mangle_unused_area_complete() {
51 // This method should do nothing.
52 // It can be called on a numa space during a full compaction.
53 }
54 void MutableNUMASpace::mangle_region(MemRegion mr) {
55 // This method should do nothing because numa spaces are not mangled.
56 }
57 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
58 assert(false, "Do not mangle MutableNUMASpace's");
59 }
60 void MutableNUMASpace::set_top_for_allocations() {
61 // This method should do nothing.
62 }
63 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
64 // This method should do nothing.
65 }
66 void MutableNUMASpace::check_mangled_unused_area_complete() {
67 // This method should do nothing.
68 }
69 #endif // NOT_PRODUCT
71 // There may be unallocated holes in the middle chunks
72 // that should be filled with dead objects to ensure parseability.
73 void MutableNUMASpace::ensure_parsability() {
74 for (int i = 0; i < lgrp_spaces()->length(); i++) {
75 LGRPSpace *ls = lgrp_spaces()->at(i);
76 MutableSpace *s = ls->space();
77 if (s->top() < top()) { // For all spaces preceeding the one containing top()
78 if (s->free_in_words() > 0) {
79 SharedHeap::fill_region_with_object(MemRegion(s->top(), s->end()));
80 size_t area_touched_words = pointer_delta(s->end(), s->top());
81 #ifndef ASSERT
82 if (!ZapUnusedHeapArea) {
83 area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
84 area_touched_words);
85 }
86 #endif
87 if (!os::numa_has_static_binding()) {
88 MemRegion invalid;
89 HeapWord *crossing_start = (HeapWord*)round_to((intptr_t)s->top(), os::vm_page_size());
90 HeapWord *crossing_end = (HeapWord*)round_to((intptr_t)(s->top() + area_touched_words),
91 os::vm_page_size());
92 if (crossing_start != crossing_end) {
93 // If object header crossed a small page boundary we mark the area
94 // as invalid rounding it to a page_size().
95 HeapWord *start = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
96 HeapWord *end = MIN2((HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), page_size()),
97 s->end());
98 invalid = MemRegion(start, end);
99 }
101 ls->add_invalid_region(invalid);
102 }
103 }
104 } else {
105 if (!os::numa_has_static_binding()) {
106 #ifdef ASSERT
107 MemRegion invalid(s->top(), s->end());
108 ls->add_invalid_region(invalid);
109 #else
110 if (ZapUnusedHeapArea) {
111 MemRegion invalid(s->top(), s->end());
112 ls->add_invalid_region(invalid);
113 } else {
114 return;
115 }
116 #endif
117 } else {
118 return;
119 }
120 }
121 }
122 }
124 size_t MutableNUMASpace::used_in_words() const {
125 size_t s = 0;
126 for (int i = 0; i < lgrp_spaces()->length(); i++) {
127 s += lgrp_spaces()->at(i)->space()->used_in_words();
128 }
129 return s;
130 }
132 size_t MutableNUMASpace::free_in_words() const {
133 size_t s = 0;
134 for (int i = 0; i < lgrp_spaces()->length(); i++) {
135 s += lgrp_spaces()->at(i)->space()->free_in_words();
136 }
137 return s;
138 }
141 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
142 guarantee(thr != NULL, "No thread");
143 int lgrp_id = thr->lgrp_id();
144 if (lgrp_id == -1) {
145 // This case can occur after the topology of the system has
146 // changed. Thread can change their location, the new home
147 // group will be determined during the first allocation
148 // attempt. For now we can safely assume that all spaces
149 // have equal size because the whole space will be reinitialized.
150 if (lgrp_spaces()->length() > 0) {
151 return capacity_in_bytes() / lgrp_spaces()->length();
152 } else {
153 assert(false, "There should be at least one locality group");
154 return 0;
155 }
156 }
157 // That's the normal case, where we know the locality group of the thread.
158 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
159 if (i == -1) {
160 return 0;
161 }
162 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
163 }
165 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
166 // Please see the comments for tlab_capacity().
167 guarantee(thr != NULL, "No thread");
168 int lgrp_id = thr->lgrp_id();
169 if (lgrp_id == -1) {
170 if (lgrp_spaces()->length() > 0) {
171 return free_in_bytes() / lgrp_spaces()->length();
172 } else {
173 assert(false, "There should be at least one locality group");
174 return 0;
175 }
176 }
177 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
178 if (i == -1) {
179 return 0;
180 }
181 return lgrp_spaces()->at(i)->space()->free_in_bytes();
182 }
185 size_t MutableNUMASpace::capacity_in_words(Thread* thr) const {
186 guarantee(thr != NULL, "No thread");
187 int lgrp_id = thr->lgrp_id();
188 if (lgrp_id == -1) {
189 if (lgrp_spaces()->length() > 0) {
190 return capacity_in_words() / lgrp_spaces()->length();
191 } else {
192 assert(false, "There should be at least one locality group");
193 return 0;
194 }
195 }
196 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
197 if (i == -1) {
198 return 0;
199 }
200 return lgrp_spaces()->at(i)->space()->capacity_in_words();
201 }
203 // Check if the NUMA topology has changed. Add and remove spaces if needed.
204 // The update can be forced by setting the force parameter equal to true.
205 bool MutableNUMASpace::update_layout(bool force) {
206 // Check if the topology had changed.
207 bool changed = os::numa_topology_changed();
208 if (force || changed) {
209 // Compute lgrp intersection. Add/remove spaces.
210 int lgrp_limit = (int)os::numa_get_groups_num();
211 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit);
212 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
213 assert(lgrp_num > 0, "There should be at least one locality group");
214 // Add new spaces for the new nodes
215 for (int i = 0; i < lgrp_num; i++) {
216 bool found = false;
217 for (int j = 0; j < lgrp_spaces()->length(); j++) {
218 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
219 found = true;
220 break;
221 }
222 }
223 if (!found) {
224 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i]));
225 }
226 }
228 // Remove spaces for the removed nodes.
229 for (int i = 0; i < lgrp_spaces()->length();) {
230 bool found = false;
231 for (int j = 0; j < lgrp_num; j++) {
232 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
233 found = true;
234 break;
235 }
236 }
237 if (!found) {
238 delete lgrp_spaces()->at(i);
239 lgrp_spaces()->remove_at(i);
240 } else {
241 i++;
242 }
243 }
245 FREE_C_HEAP_ARRAY(int, lgrp_ids);
247 if (changed) {
248 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
249 thread->set_lgrp_id(-1);
250 }
251 }
252 return true;
253 }
254 return false;
255 }
257 // Bias region towards the first-touching lgrp. Set the right page sizes.
258 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
259 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
260 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
261 if (end > start) {
262 MemRegion aligned_region(start, end);
263 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
264 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
265 assert(region().contains(aligned_region), "Sanity");
266 // First we tell the OS which page size we want in the given range. The underlying
267 // large page can be broken down if we require small pages.
268 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
269 // Then we uncommit the pages in the range.
270 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size());
271 // And make them local/first-touch biased.
272 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
273 }
274 }
276 // Free all pages in the region.
277 void MutableNUMASpace::free_region(MemRegion mr) {
278 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
279 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
280 if (end > start) {
281 MemRegion aligned_region(start, end);
282 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
283 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
284 assert(region().contains(aligned_region), "Sanity");
285 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size());
286 }
287 }
289 // Update space layout. Perform adaptation.
290 void MutableNUMASpace::update() {
291 if (update_layout(false)) {
292 // If the topology has changed, make all chunks zero-sized.
293 // And clear the alloc-rate statistics.
294 // In future we may want to handle this more gracefully in order
295 // to avoid the reallocation of the pages as much as possible.
296 for (int i = 0; i < lgrp_spaces()->length(); i++) {
297 LGRPSpace *ls = lgrp_spaces()->at(i);
298 MutableSpace *s = ls->space();
299 s->set_end(s->bottom());
300 s->set_top(s->bottom());
301 ls->clear_alloc_rate();
302 }
303 // A NUMA space is never mangled
304 initialize(region(),
305 SpaceDecorator::Clear,
306 SpaceDecorator::DontMangle);
307 } else {
308 bool should_initialize = false;
309 if (!os::numa_has_static_binding()) {
310 for (int i = 0; i < lgrp_spaces()->length(); i++) {
311 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
312 should_initialize = true;
313 break;
314 }
315 }
316 }
318 if (should_initialize ||
319 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
320 // A NUMA space is never mangled
321 initialize(region(),
322 SpaceDecorator::Clear,
323 SpaceDecorator::DontMangle);
324 }
325 }
327 if (NUMAStats) {
328 for (int i = 0; i < lgrp_spaces()->length(); i++) {
329 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
330 }
331 }
333 scan_pages(NUMAPageScanRate);
334 }
336 // Scan pages. Free pages that have smaller size or wrong placement.
337 void MutableNUMASpace::scan_pages(size_t page_count)
338 {
339 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
340 if (pages_per_chunk > 0) {
341 for (int i = 0; i < lgrp_spaces()->length(); i++) {
342 LGRPSpace *ls = lgrp_spaces()->at(i);
343 ls->scan_pages(page_size(), pages_per_chunk);
344 }
345 }
346 }
348 // Accumulate statistics about the allocation rate of each lgrp.
349 void MutableNUMASpace::accumulate_statistics() {
350 if (UseAdaptiveNUMAChunkSizing) {
351 for (int i = 0; i < lgrp_spaces()->length(); i++) {
352 lgrp_spaces()->at(i)->sample();
353 }
354 increment_samples_count();
355 }
357 if (NUMAStats) {
358 for (int i = 0; i < lgrp_spaces()->length(); i++) {
359 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
360 }
361 }
362 }
364 // Get the current size of a chunk.
365 // This function computes the size of the chunk based on the
366 // difference between chunk ends. This allows it to work correctly in
367 // case the whole space is resized and during the process of adaptive
368 // chunk resizing.
369 size_t MutableNUMASpace::current_chunk_size(int i) {
370 HeapWord *cur_end, *prev_end;
371 if (i == 0) {
372 prev_end = bottom();
373 } else {
374 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
375 }
376 if (i == lgrp_spaces()->length() - 1) {
377 cur_end = end();
378 } else {
379 cur_end = lgrp_spaces()->at(i)->space()->end();
380 }
381 if (cur_end > prev_end) {
382 return pointer_delta(cur_end, prev_end, sizeof(char));
383 }
384 return 0;
385 }
387 // Return the default chunk size by equally diving the space.
388 // page_size() aligned.
389 size_t MutableNUMASpace::default_chunk_size() {
390 return base_space_size() / lgrp_spaces()->length() * page_size();
391 }
393 // Produce a new chunk size. page_size() aligned.
394 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
395 size_t pages_available = base_space_size();
396 for (int j = 0; j < i; j++) {
397 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
398 }
399 pages_available -= lgrp_spaces()->length() - i - 1;
400 assert(pages_available > 0, "No pages left");
401 float alloc_rate = 0;
402 for (int j = i; j < lgrp_spaces()->length(); j++) {
403 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
404 }
405 size_t chunk_size = 0;
406 if (alloc_rate > 0) {
407 LGRPSpace *ls = lgrp_spaces()->at(i);
408 chunk_size = (size_t)(ls->alloc_rate()->average() * pages_available / alloc_rate) * page_size();
409 }
410 chunk_size = MAX2(chunk_size, page_size());
412 if (limit > 0) {
413 limit = round_down(limit, page_size());
414 if (chunk_size > current_chunk_size(i)) {
415 chunk_size = MIN2((off_t)chunk_size, (off_t)current_chunk_size(i) + (off_t)limit);
416 } else {
417 chunk_size = MAX2((off_t)chunk_size, (off_t)current_chunk_size(i) - (off_t)limit);
418 }
419 }
420 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
421 return chunk_size;
422 }
425 // Return the bottom_region and the top_region. Align them to page_size() boundary.
426 // |------------------new_region---------------------------------|
427 // |----bottom_region--|---intersection---|------top_region------|
428 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
429 MemRegion* bottom_region, MemRegion *top_region) {
430 // Is there bottom?
431 if (new_region.start() < intersection.start()) { // Yes
432 // Try to coalesce small pages into a large one.
433 if (UseLargePages && page_size() >= os::large_page_size()) {
434 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), os::large_page_size());
435 if (new_region.contains(p)
436 && pointer_delta(p, new_region.start(), sizeof(char)) >= os::large_page_size()) {
437 if (intersection.contains(p)) {
438 intersection = MemRegion(p, intersection.end());
439 } else {
440 intersection = MemRegion(p, p);
441 }
442 }
443 }
444 *bottom_region = MemRegion(new_region.start(), intersection.start());
445 } else {
446 *bottom_region = MemRegion();
447 }
449 // Is there top?
450 if (intersection.end() < new_region.end()) { // Yes
451 // Try to coalesce small pages into a large one.
452 if (UseLargePages && page_size() >= os::large_page_size()) {
453 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), os::large_page_size());
454 if (new_region.contains(p)
455 && pointer_delta(new_region.end(), p, sizeof(char)) >= os::large_page_size()) {
456 if (intersection.contains(p)) {
457 intersection = MemRegion(intersection.start(), p);
458 } else {
459 intersection = MemRegion(p, p);
460 }
461 }
462 }
463 *top_region = MemRegion(intersection.end(), new_region.end());
464 } else {
465 *top_region = MemRegion();
466 }
467 }
469 // Try to merge the invalid region with the bottom or top region by decreasing
470 // the intersection area. Return the invalid_region aligned to the page_size()
471 // boundary if it's inside the intersection. Return non-empty invalid_region
472 // if it lies inside the intersection (also page-aligned).
473 // |------------------new_region---------------------------------|
474 // |----------------|-------invalid---|--------------------------|
475 // |----bottom_region--|---intersection---|------top_region------|
476 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
477 MemRegion *invalid_region) {
478 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
479 *intersection = MemRegion(invalid_region->end(), intersection->end());
480 *invalid_region = MemRegion();
481 } else
482 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
483 *intersection = MemRegion(intersection->start(), invalid_region->start());
484 *invalid_region = MemRegion();
485 } else
486 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
487 *intersection = MemRegion(new_region.start(), new_region.start());
488 *invalid_region = MemRegion();
489 } else
490 if (intersection->contains(invalid_region)) {
491 // That's the only case we have to make an additional bias_region() call.
492 HeapWord* start = invalid_region->start();
493 HeapWord* end = invalid_region->end();
494 if (UseLargePages && page_size() >= os::large_page_size()) {
495 HeapWord *p = (HeapWord*)round_down((intptr_t) start, os::large_page_size());
496 if (new_region.contains(p)) {
497 start = p;
498 }
499 p = (HeapWord*)round_to((intptr_t) end, os::large_page_size());
500 if (new_region.contains(end)) {
501 end = p;
502 }
503 }
504 if (intersection->start() > start) {
505 *intersection = MemRegion(start, intersection->end());
506 }
507 if (intersection->end() < end) {
508 *intersection = MemRegion(intersection->start(), end);
509 }
510 *invalid_region = MemRegion(start, end);
511 }
512 }
514 void MutableNUMASpace::initialize(MemRegion mr,
515 bool clear_space,
516 bool mangle_space) {
517 assert(clear_space, "Reallocation will destory data!");
518 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
520 MemRegion old_region = region(), new_region;
521 set_bottom(mr.start());
522 set_end(mr.end());
523 // Must always clear the space
524 clear(SpaceDecorator::DontMangle);
526 // Compute chunk sizes
527 size_t prev_page_size = page_size();
528 set_page_size(UseLargePages ? os::large_page_size() : os::vm_page_size());
529 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
530 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
531 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
533 // Try small pages if the chunk size is too small
534 if (base_space_size_pages / lgrp_spaces()->length() == 0
535 && page_size() > (size_t)os::vm_page_size()) {
536 set_page_size(os::vm_page_size());
537 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
538 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
539 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
540 }
541 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
542 set_base_space_size(base_space_size_pages);
544 // Handle space resize
545 MemRegion top_region, bottom_region;
546 if (!old_region.equals(region())) {
547 new_region = MemRegion(rounded_bottom, rounded_end);
548 MemRegion intersection = new_region.intersection(old_region);
549 if (intersection.start() == NULL ||
550 intersection.end() == NULL ||
551 prev_page_size > page_size()) { // If the page size got smaller we have to change
552 // the page size preference for the whole space.
553 intersection = MemRegion(new_region.start(), new_region.start());
554 }
555 select_tails(new_region, intersection, &bottom_region, &top_region);
556 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
557 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
558 }
560 // Check if the space layout has changed significantly?
561 // This happens when the space has been resized so that either head or tail
562 // chunk became less than a page.
563 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
564 current_chunk_size(0) > page_size() &&
565 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
568 for (int i = 0; i < lgrp_spaces()->length(); i++) {
569 LGRPSpace *ls = lgrp_spaces()->at(i);
570 MutableSpace *s = ls->space();
571 old_region = s->region();
573 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
574 if (i < lgrp_spaces()->length() - 1) {
575 if (!UseAdaptiveNUMAChunkSizing ||
576 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
577 samples_count() < AdaptiveSizePolicyReadyThreshold) {
578 // No adaptation. Divide the space equally.
579 chunk_byte_size = default_chunk_size();
580 } else
581 if (!layout_valid || NUMASpaceResizeRate == 0) {
582 // Fast adaptation. If no space resize rate is set, resize
583 // the chunks instantly.
584 chunk_byte_size = adaptive_chunk_size(i, 0);
585 } else {
586 // Slow adaptation. Resize the chunks moving no more than
587 // NUMASpaceResizeRate bytes per collection.
588 size_t limit = NUMASpaceResizeRate /
589 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
590 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
591 }
593 assert(chunk_byte_size >= page_size(), "Chunk size too small");
594 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
595 }
597 if (i == 0) { // Bottom chunk
598 if (i != lgrp_spaces()->length() - 1) {
599 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
600 } else {
601 new_region = MemRegion(bottom(), end());
602 }
603 } else
604 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
605 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
606 new_region = MemRegion(ps->end(),
607 ps->end() + (chunk_byte_size >> LogHeapWordSize));
608 } else { // Top chunk
609 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
610 new_region = MemRegion(ps->end(), end());
611 }
612 guarantee(region().contains(new_region), "Region invariant");
615 // The general case:
616 // |---------------------|--invalid---|--------------------------|
617 // |------------------new_region---------------------------------|
618 // |----bottom_region--|---intersection---|------top_region------|
619 // |----old_region----|
620 // The intersection part has all pages in place we don't need to migrate them.
621 // Pages for the top and bottom part should be freed and then reallocated.
623 MemRegion intersection = old_region.intersection(new_region);
625 if (intersection.start() == NULL || intersection.end() == NULL) {
626 intersection = MemRegion(new_region.start(), new_region.start());
627 }
629 if (!os::numa_has_static_binding()) {
630 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
631 // Invalid region is a range of memory that could've possibly
632 // been allocated on the other node. That's relevant only on Solaris where
633 // there is no static memory binding.
634 if (!invalid_region.is_empty()) {
635 merge_regions(new_region, &intersection, &invalid_region);
636 free_region(invalid_region);
637 ls->set_invalid_region(MemRegion());
638 }
639 }
641 select_tails(new_region, intersection, &bottom_region, &top_region);
643 if (!os::numa_has_static_binding()) {
644 // If that's a system with the first-touch policy then it's enough
645 // to free the pages.
646 free_region(bottom_region);
647 free_region(top_region);
648 } else {
649 // In a system with static binding we have to change the bias whenever
650 // we reshape the heap.
651 bias_region(bottom_region, ls->lgrp_id());
652 bias_region(top_region, ls->lgrp_id());
653 }
655 // Clear space (set top = bottom) but never mangle.
656 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle);
658 set_adaptation_cycles(samples_count());
659 }
660 }
662 // Set the top of the whole space.
663 // Mark the the holes in chunks below the top() as invalid.
664 void MutableNUMASpace::set_top(HeapWord* value) {
665 bool found_top = false;
666 for (int i = 0; i < lgrp_spaces()->length();) {
667 LGRPSpace *ls = lgrp_spaces()->at(i);
668 MutableSpace *s = ls->space();
669 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
671 if (s->contains(value)) {
672 // Check if setting the chunk's top to a given value would create a hole less than
673 // a minimal object; assuming that's not the last chunk in which case we don't care.
674 if (i < lgrp_spaces()->length() - 1) {
675 size_t remainder = pointer_delta(s->end(), value);
676 const size_t minimal_object_size = oopDesc::header_size();
677 if (remainder < minimal_object_size && remainder > 0) {
678 // Add a filler object of a minimal size, it will cross the chunk boundary.
679 SharedHeap::fill_region_with_object(MemRegion(value, minimal_object_size));
680 value += minimal_object_size;
681 assert(!s->contains(value), "Should be in the next chunk");
682 // Restart the loop from the same chunk, since the value has moved
683 // to the next one.
684 continue;
685 }
686 }
688 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
689 ls->add_invalid_region(MemRegion(top, value));
690 }
691 s->set_top(value);
692 found_top = true;
693 } else {
694 if (found_top) {
695 s->set_top(s->bottom());
696 } else {
697 if (!os::numa_has_static_binding() && top < s->end()) {
698 ls->add_invalid_region(MemRegion(top, s->end()));
699 }
700 s->set_top(s->end());
701 }
702 }
703 i++;
704 }
705 MutableSpace::set_top(value);
706 }
708 void MutableNUMASpace::clear(bool mangle_space) {
709 MutableSpace::set_top(bottom());
710 for (int i = 0; i < lgrp_spaces()->length(); i++) {
711 // Never mangle NUMA spaces because the mangling will
712 // bind the memory to a possibly unwanted lgroup.
713 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
714 }
715 }
717 /*
718 Linux supports static memory binding, therefore the most part of the
719 logic dealing with the possible invalid page allocation is effectively
720 disabled. Besides there is no notion of the home node in Linux. A
721 thread is allowed to migrate freely. Although the scheduler is rather
722 reluctant to move threads between the nodes. We check for the current
723 node every allocation. And with a high probability a thread stays on
724 the same node for some time allowing local access to recently allocated
725 objects.
726 */
728 HeapWord* MutableNUMASpace::allocate(size_t size) {
729 Thread* thr = Thread::current();
730 int lgrp_id = thr->lgrp_id();
731 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
732 lgrp_id = os::numa_get_group_id();
733 thr->set_lgrp_id(lgrp_id);
734 }
736 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
738 // It is possible that a new CPU has been hotplugged and
739 // we haven't reshaped the space accordingly.
740 if (i == -1) {
741 i = os::random() % lgrp_spaces()->length();
742 }
744 LGRPSpace* ls = lgrp_spaces()->at(i);
745 MutableSpace *s = ls->space();
746 HeapWord *p = s->allocate(size);
748 if (p != NULL) {
749 size_t remainder = s->free_in_words();
750 if (remainder < (size_t)oopDesc::header_size() && remainder > 0) {
751 s->set_top(s->top() - size);
752 p = NULL;
753 }
754 }
755 if (p != NULL) {
756 if (top() < s->top()) { // Keep _top updated.
757 MutableSpace::set_top(s->top());
758 }
759 }
760 // Make the page allocation happen here if there is no static binding..
761 if (p != NULL && !os::numa_has_static_binding()) {
762 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
763 *(int*)i = 0;
764 }
765 }
766 if (p == NULL) {
767 ls->set_allocation_failed();
768 }
769 return p;
770 }
772 // This version is lock-free.
773 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
774 Thread* thr = Thread::current();
775 int lgrp_id = thr->lgrp_id();
776 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
777 lgrp_id = os::numa_get_group_id();
778 thr->set_lgrp_id(lgrp_id);
779 }
781 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
782 // It is possible that a new CPU has been hotplugged and
783 // we haven't reshaped the space accordingly.
784 if (i == -1) {
785 i = os::random() % lgrp_spaces()->length();
786 }
787 LGRPSpace *ls = lgrp_spaces()->at(i);
788 MutableSpace *s = ls->space();
789 HeapWord *p = s->cas_allocate(size);
790 if (p != NULL) {
791 size_t remainder = pointer_delta(s->end(), p + size);
792 if (remainder < (size_t)oopDesc::header_size() && remainder > 0) {
793 if (s->cas_deallocate(p, size)) {
794 // We were the last to allocate and created a fragment less than
795 // a minimal object.
796 p = NULL;
797 } else {
798 guarantee(false, "Deallocation should always succeed");
799 }
800 }
801 }
802 if (p != NULL) {
803 HeapWord* cur_top, *cur_chunk_top = p + size;
804 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
805 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
806 break;
807 }
808 }
809 }
811 // Make the page allocation happen here if there is no static binding.
812 if (p != NULL && !os::numa_has_static_binding() ) {
813 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
814 *(int*)i = 0;
815 }
816 }
817 if (p == NULL) {
818 ls->set_allocation_failed();
819 }
820 return p;
821 }
823 void MutableNUMASpace::print_short_on(outputStream* st) const {
824 MutableSpace::print_short_on(st);
825 st->print(" (");
826 for (int i = 0; i < lgrp_spaces()->length(); i++) {
827 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
828 lgrp_spaces()->at(i)->space()->print_short_on(st);
829 if (i < lgrp_spaces()->length() - 1) {
830 st->print(", ");
831 }
832 }
833 st->print(")");
834 }
836 void MutableNUMASpace::print_on(outputStream* st) const {
837 MutableSpace::print_on(st);
838 for (int i = 0; i < lgrp_spaces()->length(); i++) {
839 LGRPSpace *ls = lgrp_spaces()->at(i);
840 st->print(" lgrp %d", ls->lgrp_id());
841 ls->space()->print_on(st);
842 if (NUMAStats) {
843 for (int i = 0; i < lgrp_spaces()->length(); i++) {
844 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
845 }
846 st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n",
847 ls->space_stats()->_local_space / K,
848 ls->space_stats()->_remote_space / K,
849 ls->space_stats()->_unbiased_space / K,
850 ls->space_stats()->_uncommited_space / K,
851 ls->space_stats()->_large_pages,
852 ls->space_stats()->_small_pages);
853 }
854 }
855 }
857 void MutableNUMASpace::verify(bool allow_dirty) {
858 // This can be called after setting an arbitary value to the space's top,
859 // so an object can cross the chunk boundary. We ensure the parsablity
860 // of the space and just walk the objects in linear fashion.
861 ensure_parsability();
862 MutableSpace::verify(allow_dirty);
863 }
865 // Scan pages and gather stats about page placement and size.
866 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
867 clear_space_stats();
868 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
869 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
870 if (start < end) {
871 for (char *p = start; p < end;) {
872 os::page_info info;
873 if (os::get_page_info(p, &info)) {
874 if (info.size > 0) {
875 if (info.size > (size_t)os::vm_page_size()) {
876 space_stats()->_large_pages++;
877 } else {
878 space_stats()->_small_pages++;
879 }
880 if (info.lgrp_id == lgrp_id()) {
881 space_stats()->_local_space += info.size;
882 } else {
883 space_stats()->_remote_space += info.size;
884 }
885 p += info.size;
886 } else {
887 p += os::vm_page_size();
888 space_stats()->_uncommited_space += os::vm_page_size();
889 }
890 } else {
891 return;
892 }
893 }
894 }
895 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
896 pointer_delta(space()->end(), end, sizeof(char));
898 }
900 // Scan page_count pages and verify if they have the right size and right placement.
901 // If invalid pages are found they are freed in hope that subsequent reallocation
902 // will be more successful.
903 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
904 {
905 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
906 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
908 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
909 set_last_page_scanned(range_start);
910 }
912 char *scan_start = last_page_scanned();
913 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
915 os::page_info page_expected, page_found;
916 page_expected.size = page_size;
917 page_expected.lgrp_id = lgrp_id();
919 char *s = scan_start;
920 while (s < scan_end) {
921 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
922 if (e == NULL) {
923 break;
924 }
925 if (e != scan_end) {
926 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
927 && page_expected.size != 0) {
928 os::free_memory(s, pointer_delta(e, s, sizeof(char)));
929 }
930 page_expected = page_found;
931 }
932 s = e;
933 }
935 set_last_page_scanned(scan_end);
936 }