Mercurial > jdk8-mips64-public > hotspot / file revision
src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp@d6340ab4105b
src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp
Thu, 17 Jul 2008 10:26:33 -0700
- author
- iveresov
- date
- Thu, 17 Jul 2008 10:26:33 -0700
- changeset 703
- d6340ab4105b
- parent 698
- 12eea04c8b06
- child 704
- 850fdf70db2b
- permissions
- -rw-r--r--
6723228: NUMA allocator: assert(lgrp_id != -1, "No lgrp_id set")
6723229: NUMA allocator: assert(lgrp_num > 0, "There should be at least one locality group")
Summary: The fix takes care of the assertion triggered during TLAB resizing after reconfiguration. Also it now handles a defect in the topology graph, in which a single leaf node doesn't have memory.
Reviewed-by: jmasa
2 /*
3 * Copyright 2006-2007 Sun Microsystems, Inc. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
22 * have any questions.
23 *
24 */
26 # include "incls/_precompiled.incl"
27 # include "incls/_mutableNUMASpace.cpp.incl"
30 MutableNUMASpace::MutableNUMASpace() {
31 _lgrp_spaces = new (ResourceObj::C_HEAP) GrowableArray<LGRPSpace*>(0, true);
32 _page_size = os::vm_page_size();
33 _adaptation_cycles = 0;
34 _samples_count = 0;
35 update_layout(true);
36 }
38 MutableNUMASpace::~MutableNUMASpace() {
39 for (int i = 0; i < lgrp_spaces()->length(); i++) {
40 delete lgrp_spaces()->at(i);
41 }
42 delete lgrp_spaces();
43 }
45 #ifndef PRODUCT
46 void MutableNUMASpace::mangle_unused_area() {
47 // This method should do nothing.
48 // It can be called on a numa space during a full compaction.
49 }
50 void MutableNUMASpace::mangle_unused_area_complete() {
51 // This method should do nothing.
52 // It can be called on a numa space during a full compaction.
53 }
54 void MutableNUMASpace::mangle_region(MemRegion mr) {
55 // This method should do nothing because numa spaces are not mangled.
56 }
57 void MutableNUMASpace::set_top_for_allocations(HeapWord* v) {
58 assert(false, "Do not mangle MutableNUMASpace's");
59 }
60 void MutableNUMASpace::set_top_for_allocations() {
61 // This method should do nothing.
62 }
63 void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) {
64 // This method should do nothing.
65 }
66 void MutableNUMASpace::check_mangled_unused_area_complete() {
67 // This method should do nothing.
68 }
69 #endif // NOT_PRODUCT
71 // There may be unallocated holes in the middle chunks
72 // that should be filled with dead objects to ensure parseability.
73 void MutableNUMASpace::ensure_parsability() {
74 for (int i = 0; i < lgrp_spaces()->length(); i++) {
75 LGRPSpace *ls = lgrp_spaces()->at(i);
76 MutableSpace *s = ls->space();
77 if (s->top() < top()) { // For all spaces preceeding the one containing top()
78 if (s->free_in_words() > 0) {
79 SharedHeap::fill_region_with_object(MemRegion(s->top(), s->end()));
80 size_t area_touched_words = pointer_delta(s->end(), s->top());
81 #ifndef ASSERT
82 if (!ZapUnusedHeapArea) {
83 area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
84 area_touched_words);
85 }
86 #endif
87 if (!os::numa_has_static_binding()) {
88 MemRegion invalid;
89 HeapWord *crossing_start = (HeapWord*)round_to((intptr_t)s->top(), os::vm_page_size());
90 HeapWord *crossing_end = (HeapWord*)round_to((intptr_t)(s->top() + area_touched_words),
91 os::vm_page_size());
92 if (crossing_start != crossing_end) {
93 // If object header crossed a small page boundary we mark the area
94 // as invalid rounding it to a page_size().
95 HeapWord *start = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
96 HeapWord *end = MIN2((HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), page_size()),
97 s->end());
98 invalid = MemRegion(start, end);
99 }
101 ls->add_invalid_region(invalid);
102 }
103 }
104 } else {
105 if (!os::numa_has_static_binding()) {
106 #ifdef ASSERT
107 MemRegion invalid(s->top(), s->end());
108 ls->add_invalid_region(invalid);
109 #else
110 if (ZapUnusedHeapArea) {
111 MemRegion invalid(s->top(), s->end());
112 ls->add_invalid_region(invalid);
113 } else {
114 return;
115 }
116 #endif
117 } else {
118 return;
119 }
120 }
121 }
122 }
124 size_t MutableNUMASpace::used_in_words() const {
125 size_t s = 0;
126 for (int i = 0; i < lgrp_spaces()->length(); i++) {
127 s += lgrp_spaces()->at(i)->space()->used_in_words();
128 }
129 return s;
130 }
132 size_t MutableNUMASpace::free_in_words() const {
133 size_t s = 0;
134 for (int i = 0; i < lgrp_spaces()->length(); i++) {
135 s += lgrp_spaces()->at(i)->space()->free_in_words();
136 }
137 return s;
138 }
141 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
142 guarantee(thr != NULL, "No thread");
143 int lgrp_id = thr->lgrp_id();
144 if (lgrp_id == -1) {
145 // This case can occur after the topology of the system has
146 // changed. Thread can change their location, the new home
147 // group will be determined during the first allocation
148 // attempt. For now we can safely assume that all spaces
149 // have equal size because the whole space will be reinitialized.
150 if (lgrp_spaces()->length() > 0) {
151 return capacity_in_bytes() / lgrp_spaces()->length();
152 } else {
153 assert(false, "There should be at least one locality group");
154 return 0;
155 }
156 }
157 // That's the normal case, where we know the locality group of the thread.
158 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
159 if (i == -1) {
160 return 0;
161 }
162 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
163 }
165 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
166 // Please see the comments for tlab_capacity().
167 guarantee(thr != NULL, "No thread");
168 int lgrp_id = thr->lgrp_id();
169 if (lgrp_id == -1) {
170 if (lgrp_spaces()->length() > 0) {
171 return free_in_bytes() / lgrp_spaces()->length();
172 } else {
173 assert(false, "There should be at least one locality group");
174 return 0;
175 }
176 }
177 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
178 if (i == -1) {
179 return 0;
180 }
181 return lgrp_spaces()->at(i)->space()->free_in_bytes();
182 }
184 // Check if the NUMA topology has changed. Add and remove spaces if needed.
185 // The update can be forced by setting the force parameter equal to true.
186 bool MutableNUMASpace::update_layout(bool force) {
187 // Check if the topology had changed.
188 bool changed = os::numa_topology_changed();
189 if (force || changed) {
190 // Compute lgrp intersection. Add/remove spaces.
191 int lgrp_limit = (int)os::numa_get_groups_num();
192 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit);
193 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
194 assert(lgrp_num > 0, "There should be at least one locality group");
195 // Add new spaces for the new nodes
196 for (int i = 0; i < lgrp_num; i++) {
197 bool found = false;
198 for (int j = 0; j < lgrp_spaces()->length(); j++) {
199 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
200 found = true;
201 break;
202 }
203 }
204 if (!found) {
205 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i]));
206 }
207 }
209 // Remove spaces for the removed nodes.
210 for (int i = 0; i < lgrp_spaces()->length();) {
211 bool found = false;
212 for (int j = 0; j < lgrp_num; j++) {
213 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
214 found = true;
215 break;
216 }
217 }
218 if (!found) {
219 delete lgrp_spaces()->at(i);
220 lgrp_spaces()->remove_at(i);
221 } else {
222 i++;
223 }
224 }
226 FREE_C_HEAP_ARRAY(int, lgrp_ids);
228 if (changed) {
229 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
230 thread->set_lgrp_id(-1);
231 }
232 }
233 return true;
234 }
235 return false;
236 }
238 // Bias region towards the first-touching lgrp. Set the right page sizes.
239 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
240 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
241 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
242 if (end > start) {
243 MemRegion aligned_region(start, end);
244 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
245 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
246 assert(region().contains(aligned_region), "Sanity");
247 // First we tell the OS which page size we want in the given range. The underlying
248 // large page can be broken down if we require small pages.
249 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
250 // Then we uncommit the pages in the range.
251 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size());
252 // And make them local/first-touch biased.
253 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
254 }
255 }
257 // Free all pages in the region.
258 void MutableNUMASpace::free_region(MemRegion mr) {
259 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
260 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
261 if (end > start) {
262 MemRegion aligned_region(start, end);
263 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
264 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
265 assert(region().contains(aligned_region), "Sanity");
266 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size());
267 }
268 }
270 // Update space layout. Perform adaptation.
271 void MutableNUMASpace::update() {
272 if (update_layout(false)) {
273 // If the topology has changed, make all chunks zero-sized.
274 // And clear the alloc-rate statistics.
275 // In future we may want to handle this more gracefully in order
276 // to avoid the reallocation of the pages as much as possible.
277 for (int i = 0; i < lgrp_spaces()->length(); i++) {
278 LGRPSpace *ls = lgrp_spaces()->at(i);
279 MutableSpace *s = ls->space();
280 s->set_end(s->bottom());
281 s->set_top(s->bottom());
282 ls->clear_alloc_rate();
283 }
284 // A NUMA space is never mangled
285 initialize(region(),
286 SpaceDecorator::Clear,
287 SpaceDecorator::DontMangle);
288 } else {
289 bool should_initialize = false;
290 if (!os::numa_has_static_binding()) {
291 for (int i = 0; i < lgrp_spaces()->length(); i++) {
292 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
293 should_initialize = true;
294 break;
295 }
296 }
297 }
299 if (should_initialize ||
300 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
301 // A NUMA space is never mangled
302 initialize(region(),
303 SpaceDecorator::Clear,
304 SpaceDecorator::DontMangle);
305 }
306 }
308 if (NUMAStats) {
309 for (int i = 0; i < lgrp_spaces()->length(); i++) {
310 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
311 }
312 }
314 scan_pages(NUMAPageScanRate);
315 }
317 // Scan pages. Free pages that have smaller size or wrong placement.
318 void MutableNUMASpace::scan_pages(size_t page_count)
319 {
320 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
321 if (pages_per_chunk > 0) {
322 for (int i = 0; i < lgrp_spaces()->length(); i++) {
323 LGRPSpace *ls = lgrp_spaces()->at(i);
324 ls->scan_pages(page_size(), pages_per_chunk);
325 }
326 }
327 }
329 // Accumulate statistics about the allocation rate of each lgrp.
330 void MutableNUMASpace::accumulate_statistics() {
331 if (UseAdaptiveNUMAChunkSizing) {
332 for (int i = 0; i < lgrp_spaces()->length(); i++) {
333 lgrp_spaces()->at(i)->sample();
334 }
335 increment_samples_count();
336 }
338 if (NUMAStats) {
339 for (int i = 0; i < lgrp_spaces()->length(); i++) {
340 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
341 }
342 }
343 }
345 // Get the current size of a chunk.
346 // This function computes the size of the chunk based on the
347 // difference between chunk ends. This allows it to work correctly in
348 // case the whole space is resized and during the process of adaptive
349 // chunk resizing.
350 size_t MutableNUMASpace::current_chunk_size(int i) {
351 HeapWord *cur_end, *prev_end;
352 if (i == 0) {
353 prev_end = bottom();
354 } else {
355 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
356 }
357 if (i == lgrp_spaces()->length() - 1) {
358 cur_end = end();
359 } else {
360 cur_end = lgrp_spaces()->at(i)->space()->end();
361 }
362 if (cur_end > prev_end) {
363 return pointer_delta(cur_end, prev_end, sizeof(char));
364 }
365 return 0;
366 }
368 // Return the default chunk size by equally diving the space.
369 // page_size() aligned.
370 size_t MutableNUMASpace::default_chunk_size() {
371 return base_space_size() / lgrp_spaces()->length() * page_size();
372 }
374 // Produce a new chunk size. page_size() aligned.
375 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
376 size_t pages_available = base_space_size();
377 for (int j = 0; j < i; j++) {
378 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
379 }
380 pages_available -= lgrp_spaces()->length() - i - 1;
381 assert(pages_available > 0, "No pages left");
382 float alloc_rate = 0;
383 for (int j = i; j < lgrp_spaces()->length(); j++) {
384 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
385 }
386 size_t chunk_size = 0;
387 if (alloc_rate > 0) {
388 LGRPSpace *ls = lgrp_spaces()->at(i);
389 chunk_size = (size_t)(ls->alloc_rate()->average() * pages_available / alloc_rate) * page_size();
390 }
391 chunk_size = MAX2(chunk_size, page_size());
393 if (limit > 0) {
394 limit = round_down(limit, page_size());
395 if (chunk_size > current_chunk_size(i)) {
396 chunk_size = MIN2((off_t)chunk_size, (off_t)current_chunk_size(i) + (off_t)limit);
397 } else {
398 chunk_size = MAX2((off_t)chunk_size, (off_t)current_chunk_size(i) - (off_t)limit);
399 }
400 }
401 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
402 return chunk_size;
403 }
406 // Return the bottom_region and the top_region. Align them to page_size() boundary.
407 // |------------------new_region---------------------------------|
408 // |----bottom_region--|---intersection---|------top_region------|
409 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
410 MemRegion* bottom_region, MemRegion *top_region) {
411 // Is there bottom?
412 if (new_region.start() < intersection.start()) { // Yes
413 // Try to coalesce small pages into a large one.
414 if (UseLargePages && page_size() >= os::large_page_size()) {
415 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), os::large_page_size());
416 if (new_region.contains(p)
417 && pointer_delta(p, new_region.start(), sizeof(char)) >= os::large_page_size()) {
418 if (intersection.contains(p)) {
419 intersection = MemRegion(p, intersection.end());
420 } else {
421 intersection = MemRegion(p, p);
422 }
423 }
424 }
425 *bottom_region = MemRegion(new_region.start(), intersection.start());
426 } else {
427 *bottom_region = MemRegion();
428 }
430 // Is there top?
431 if (intersection.end() < new_region.end()) { // Yes
432 // Try to coalesce small pages into a large one.
433 if (UseLargePages && page_size() >= os::large_page_size()) {
434 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), os::large_page_size());
435 if (new_region.contains(p)
436 && pointer_delta(new_region.end(), p, sizeof(char)) >= os::large_page_size()) {
437 if (intersection.contains(p)) {
438 intersection = MemRegion(intersection.start(), p);
439 } else {
440 intersection = MemRegion(p, p);
441 }
442 }
443 }
444 *top_region = MemRegion(intersection.end(), new_region.end());
445 } else {
446 *top_region = MemRegion();
447 }
448 }
450 // Try to merge the invalid region with the bottom or top region by decreasing
451 // the intersection area. Return the invalid_region aligned to the page_size()
452 // boundary if it's inside the intersection. Return non-empty invalid_region
453 // if it lies inside the intersection (also page-aligned).
454 // |------------------new_region---------------------------------|
455 // |----------------|-------invalid---|--------------------------|
456 // |----bottom_region--|---intersection---|------top_region------|
457 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
458 MemRegion *invalid_region) {
459 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
460 *intersection = MemRegion(invalid_region->end(), intersection->end());
461 *invalid_region = MemRegion();
462 } else
463 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
464 *intersection = MemRegion(intersection->start(), invalid_region->start());
465 *invalid_region = MemRegion();
466 } else
467 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
468 *intersection = MemRegion(new_region.start(), new_region.start());
469 *invalid_region = MemRegion();
470 } else
471 if (intersection->contains(invalid_region)) {
472 // That's the only case we have to make an additional bias_region() call.
473 HeapWord* start = invalid_region->start();
474 HeapWord* end = invalid_region->end();
475 if (UseLargePages && page_size() >= os::large_page_size()) {
476 HeapWord *p = (HeapWord*)round_down((intptr_t) start, os::large_page_size());
477 if (new_region.contains(p)) {
478 start = p;
479 }
480 p = (HeapWord*)round_to((intptr_t) end, os::large_page_size());
481 if (new_region.contains(end)) {
482 end = p;
483 }
484 }
485 if (intersection->start() > start) {
486 *intersection = MemRegion(start, intersection->end());
487 }
488 if (intersection->end() < end) {
489 *intersection = MemRegion(intersection->start(), end);
490 }
491 *invalid_region = MemRegion(start, end);
492 }
493 }
495 void MutableNUMASpace::initialize(MemRegion mr,
496 bool clear_space,
497 bool mangle_space) {
498 assert(clear_space, "Reallocation will destory data!");
499 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
501 MemRegion old_region = region(), new_region;
502 set_bottom(mr.start());
503 set_end(mr.end());
504 // Must always clear the space
505 clear(SpaceDecorator::DontMangle);
507 // Compute chunk sizes
508 size_t prev_page_size = page_size();
509 set_page_size(UseLargePages ? os::large_page_size() : os::vm_page_size());
510 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
511 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
512 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
514 // Try small pages if the chunk size is too small
515 if (base_space_size_pages / lgrp_spaces()->length() == 0
516 && page_size() > (size_t)os::vm_page_size()) {
517 set_page_size(os::vm_page_size());
518 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
519 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
520 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
521 }
522 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
523 set_base_space_size(base_space_size_pages);
525 // Handle space resize
526 MemRegion top_region, bottom_region;
527 if (!old_region.equals(region())) {
528 new_region = MemRegion(rounded_bottom, rounded_end);
529 MemRegion intersection = new_region.intersection(old_region);
530 if (intersection.start() == NULL ||
531 intersection.end() == NULL ||
532 prev_page_size > page_size()) { // If the page size got smaller we have to change
533 // the page size preference for the whole space.
534 intersection = MemRegion(new_region.start(), new_region.start());
535 }
536 select_tails(new_region, intersection, &bottom_region, &top_region);
537 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
538 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
539 }
541 // Check if the space layout has changed significantly?
542 // This happens when the space has been resized so that either head or tail
543 // chunk became less than a page.
544 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
545 current_chunk_size(0) > page_size() &&
546 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
549 for (int i = 0; i < lgrp_spaces()->length(); i++) {
550 LGRPSpace *ls = lgrp_spaces()->at(i);
551 MutableSpace *s = ls->space();
552 old_region = s->region();
554 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
555 if (i < lgrp_spaces()->length() - 1) {
556 if (!UseAdaptiveNUMAChunkSizing ||
557 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
558 samples_count() < AdaptiveSizePolicyReadyThreshold) {
559 // No adaptation. Divide the space equally.
560 chunk_byte_size = default_chunk_size();
561 } else
562 if (!layout_valid || NUMASpaceResizeRate == 0) {
563 // Fast adaptation. If no space resize rate is set, resize
564 // the chunks instantly.
565 chunk_byte_size = adaptive_chunk_size(i, 0);
566 } else {
567 // Slow adaptation. Resize the chunks moving no more than
568 // NUMASpaceResizeRate bytes per collection.
569 size_t limit = NUMASpaceResizeRate /
570 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
571 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
572 }
574 assert(chunk_byte_size >= page_size(), "Chunk size too small");
575 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
576 }
578 if (i == 0) { // Bottom chunk
579 if (i != lgrp_spaces()->length() - 1) {
580 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
581 } else {
582 new_region = MemRegion(bottom(), end());
583 }
584 } else
585 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
586 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
587 new_region = MemRegion(ps->end(),
588 ps->end() + (chunk_byte_size >> LogHeapWordSize));
589 } else { // Top chunk
590 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
591 new_region = MemRegion(ps->end(), end());
592 }
593 guarantee(region().contains(new_region), "Region invariant");
596 // The general case:
597 // |---------------------|--invalid---|--------------------------|
598 // |------------------new_region---------------------------------|
599 // |----bottom_region--|---intersection---|------top_region------|
600 // |----old_region----|
601 // The intersection part has all pages in place we don't need to migrate them.
602 // Pages for the top and bottom part should be freed and then reallocated.
604 MemRegion intersection = old_region.intersection(new_region);
606 if (intersection.start() == NULL || intersection.end() == NULL) {
607 intersection = MemRegion(new_region.start(), new_region.start());
608 }
610 if (!os::numa_has_static_binding()) {
611 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
612 // Invalid region is a range of memory that could've possibly
613 // been allocated on the other node. That's relevant only on Solaris where
614 // there is no static memory binding.
615 if (!invalid_region.is_empty()) {
616 merge_regions(new_region, &intersection, &invalid_region);
617 free_region(invalid_region);
618 ls->set_invalid_region(MemRegion());
619 }
620 }
622 select_tails(new_region, intersection, &bottom_region, &top_region);
624 if (!os::numa_has_static_binding()) {
625 // If that's a system with the first-touch policy then it's enough
626 // to free the pages.
627 free_region(bottom_region);
628 free_region(top_region);
629 } else {
630 // In a system with static binding we have to change the bias whenever
631 // we reshape the heap.
632 bias_region(bottom_region, ls->lgrp_id());
633 bias_region(top_region, ls->lgrp_id());
634 }
636 // Clear space (set top = bottom) but never mangle.
637 s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle);
639 set_adaptation_cycles(samples_count());
640 }
641 }
643 // Set the top of the whole space.
644 // Mark the the holes in chunks below the top() as invalid.
645 void MutableNUMASpace::set_top(HeapWord* value) {
646 bool found_top = false;
647 for (int i = 0; i < lgrp_spaces()->length();) {
648 LGRPSpace *ls = lgrp_spaces()->at(i);
649 MutableSpace *s = ls->space();
650 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
652 if (s->contains(value)) {
653 // Check if setting the chunk's top to a given value would create a hole less than
654 // a minimal object; assuming that's not the last chunk in which case we don't care.
655 if (i < lgrp_spaces()->length() - 1) {
656 size_t remainder = pointer_delta(s->end(), value);
657 const size_t minimal_object_size = oopDesc::header_size();
658 if (remainder < minimal_object_size && remainder > 0) {
659 // Add a filler object of a minimal size, it will cross the chunk boundary.
660 SharedHeap::fill_region_with_object(MemRegion(value, minimal_object_size));
661 value += minimal_object_size;
662 assert(!s->contains(value), "Should be in the next chunk");
663 // Restart the loop from the same chunk, since the value has moved
664 // to the next one.
665 continue;
666 }
667 }
669 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
670 ls->add_invalid_region(MemRegion(top, value));
671 }
672 s->set_top(value);
673 found_top = true;
674 } else {
675 if (found_top) {
676 s->set_top(s->bottom());
677 } else {
678 if (!os::numa_has_static_binding() && top < s->end()) {
679 ls->add_invalid_region(MemRegion(top, s->end()));
680 }
681 s->set_top(s->end());
682 }
683 }
684 i++;
685 }
686 MutableSpace::set_top(value);
687 }
689 void MutableNUMASpace::clear(bool mangle_space) {
690 MutableSpace::set_top(bottom());
691 for (int i = 0; i < lgrp_spaces()->length(); i++) {
692 // Never mangle NUMA spaces because the mangling will
693 // bind the memory to a possibly unwanted lgroup.
694 lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle);
695 }
696 }
698 /*
699 Linux supports static memory binding, therefore the most part of the
700 logic dealing with the possible invalid page allocation is effectively
701 disabled. Besides there is no notion of the home node in Linux. A
702 thread is allowed to migrate freely. Although the scheduler is rather
703 reluctant to move threads between the nodes. We check for the current
704 node every allocation. And with a high probability a thread stays on
705 the same node for some time allowing local access to recently allocated
706 objects.
707 */
709 HeapWord* MutableNUMASpace::allocate(size_t size) {
710 Thread* thr = Thread::current();
711 int lgrp_id = thr->lgrp_id();
712 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
713 lgrp_id = os::numa_get_group_id();
714 thr->set_lgrp_id(lgrp_id);
715 }
717 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
719 // It is possible that a new CPU has been hotplugged and
720 // we haven't reshaped the space accordingly.
721 if (i == -1) {
722 i = os::random() % lgrp_spaces()->length();
723 }
725 MutableSpace *s = lgrp_spaces()->at(i)->space();
726 HeapWord *p = s->allocate(size);
728 if (p != NULL) {
729 size_t remainder = s->free_in_words();
730 if (remainder < (size_t)oopDesc::header_size() && remainder > 0) {
731 s->set_top(s->top() - size);
732 p = NULL;
733 }
734 }
735 if (p != NULL) {
736 if (top() < s->top()) { // Keep _top updated.
737 MutableSpace::set_top(s->top());
738 }
739 }
740 // Make the page allocation happen here if there is no static binding..
741 if (p != NULL && !os::numa_has_static_binding()) {
742 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
743 *(int*)i = 0;
744 }
745 }
746 return p;
747 }
749 // This version is lock-free.
750 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
751 Thread* thr = Thread::current();
752 int lgrp_id = thr->lgrp_id();
753 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
754 lgrp_id = os::numa_get_group_id();
755 thr->set_lgrp_id(lgrp_id);
756 }
758 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
759 // It is possible that a new CPU has been hotplugged and
760 // we haven't reshaped the space accordingly.
761 if (i == -1) {
762 i = os::random() % lgrp_spaces()->length();
763 }
764 MutableSpace *s = lgrp_spaces()->at(i)->space();
765 HeapWord *p = s->cas_allocate(size);
766 if (p != NULL) {
767 size_t remainder = pointer_delta(s->end(), p + size);
768 if (remainder < (size_t)oopDesc::header_size() && remainder > 0) {
769 if (s->cas_deallocate(p, size)) {
770 // We were the last to allocate and created a fragment less than
771 // a minimal object.
772 p = NULL;
773 } else {
774 guarantee(false, "Deallocation should always succeed");
775 }
776 }
777 }
778 if (p != NULL) {
779 HeapWord* cur_top, *cur_chunk_top = p + size;
780 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
781 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
782 break;
783 }
784 }
785 }
787 // Make the page allocation happen here if there is no static binding.
788 if (p != NULL && !os::numa_has_static_binding() ) {
789 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
790 *(int*)i = 0;
791 }
792 }
793 return p;
794 }
796 void MutableNUMASpace::print_short_on(outputStream* st) const {
797 MutableSpace::print_short_on(st);
798 st->print(" (");
799 for (int i = 0; i < lgrp_spaces()->length(); i++) {
800 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
801 lgrp_spaces()->at(i)->space()->print_short_on(st);
802 if (i < lgrp_spaces()->length() - 1) {
803 st->print(", ");
804 }
805 }
806 st->print(")");
807 }
809 void MutableNUMASpace::print_on(outputStream* st) const {
810 MutableSpace::print_on(st);
811 for (int i = 0; i < lgrp_spaces()->length(); i++) {
812 LGRPSpace *ls = lgrp_spaces()->at(i);
813 st->print(" lgrp %d", ls->lgrp_id());
814 ls->space()->print_on(st);
815 if (NUMAStats) {
816 for (int i = 0; i < lgrp_spaces()->length(); i++) {
817 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
818 }
819 st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n",
820 ls->space_stats()->_local_space / K,
821 ls->space_stats()->_remote_space / K,
822 ls->space_stats()->_unbiased_space / K,
823 ls->space_stats()->_uncommited_space / K,
824 ls->space_stats()->_large_pages,
825 ls->space_stats()->_small_pages);
826 }
827 }
828 }
830 void MutableNUMASpace::verify(bool allow_dirty) {
831 // This can be called after setting an arbitary value to the space's top,
832 // so an object can cross the chunk boundary. We ensure the parsablity
833 // of the space and just walk the objects in linear fashion.
834 ensure_parsability();
835 MutableSpace::verify(allow_dirty);
836 }
838 // Scan pages and gather stats about page placement and size.
839 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
840 clear_space_stats();
841 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
842 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
843 if (start < end) {
844 for (char *p = start; p < end;) {
845 os::page_info info;
846 if (os::get_page_info(p, &info)) {
847 if (info.size > 0) {
848 if (info.size > (size_t)os::vm_page_size()) {
849 space_stats()->_large_pages++;
850 } else {
851 space_stats()->_small_pages++;
852 }
853 if (info.lgrp_id == lgrp_id()) {
854 space_stats()->_local_space += info.size;
855 } else {
856 space_stats()->_remote_space += info.size;
857 }
858 p += info.size;
859 } else {
860 p += os::vm_page_size();
861 space_stats()->_uncommited_space += os::vm_page_size();
862 }
863 } else {
864 return;
865 }
866 }
867 }
868 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
869 pointer_delta(space()->end(), end, sizeof(char));
871 }
873 // Scan page_count pages and verify if they have the right size and right placement.
874 // If invalid pages are found they are freed in hope that subsequent reallocation
875 // will be more successful.
876 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
877 {
878 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
879 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
881 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
882 set_last_page_scanned(range_start);
883 }
885 char *scan_start = last_page_scanned();
886 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
888 os::page_info page_expected, page_found;
889 page_expected.size = page_size;
890 page_expected.lgrp_id = lgrp_id();
892 char *s = scan_start;
893 while (s < scan_end) {
894 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
895 if (e == NULL) {
896 break;
897 }
898 if (e != scan_end) {
899 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
900 && page_expected.size != 0) {
901 os::free_memory(s, pointer_delta(e, s, sizeof(char)));
902 }
903 page_expected = page_found;
904 }
905 s = e;
906 }
908 set_last_page_scanned(scan_end);
909 }
