Thu, 05 Jun 2008 15:57:56 -0700
6711316: Open source the Garbage-First garbage collector
Summary: First mercurial integration of the code for the Garbage-First garbage collector.
Reviewed-by: apetrusenko, iveresov, jmasa, sgoldman, tonyp, ysr
2 /*
3 * Copyright 2006-2007 Sun Microsystems, Inc. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
22 * have any questions.
23 *
24 */
26 # include "incls/_precompiled.incl"
27 # include "incls/_mutableNUMASpace.cpp.incl"
30 MutableNUMASpace::MutableNUMASpace() {
31 _lgrp_spaces = new (ResourceObj::C_HEAP) GrowableArray<LGRPSpace*>(0, true);
32 _page_size = os::vm_page_size();
33 _adaptation_cycles = 0;
34 _samples_count = 0;
35 update_layout(true);
36 }
38 MutableNUMASpace::~MutableNUMASpace() {
39 for (int i = 0; i < lgrp_spaces()->length(); i++) {
40 delete lgrp_spaces()->at(i);
41 }
42 delete lgrp_spaces();
43 }
45 void MutableNUMASpace::mangle_unused_area() {
46 for (int i = 0; i < lgrp_spaces()->length(); i++) {
47 LGRPSpace *ls = lgrp_spaces()->at(i);
48 MutableSpace *s = ls->space();
49 if (!os::numa_has_static_binding()) {
50 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
51 if (top < s->end()) {
52 ls->add_invalid_region(MemRegion(top, s->end()));
53 }
54 }
55 s->mangle_unused_area();
56 }
57 }
59 // There may be unallocated holes in the middle chunks
60 // that should be filled with dead objects to ensure parseability.
61 void MutableNUMASpace::ensure_parsability() {
62 for (int i = 0; i < lgrp_spaces()->length(); i++) {
63 LGRPSpace *ls = lgrp_spaces()->at(i);
64 MutableSpace *s = ls->space();
65 if (s->top() < top()) { // For all spaces preceeding the one containing top()
66 if (s->free_in_words() > 0) {
67 SharedHeap::fill_region_with_object(MemRegion(s->top(), s->end()));
68 size_t area_touched_words = pointer_delta(s->end(), s->top());
69 #ifndef ASSERT
70 if (!ZapUnusedHeapArea) {
71 area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
72 area_touched_words);
73 }
74 #endif
75 if (!os::numa_has_static_binding()) {
76 MemRegion invalid;
77 HeapWord *crossing_start = (HeapWord*)round_to((intptr_t)s->top(), os::vm_page_size());
78 HeapWord *crossing_end = (HeapWord*)round_to((intptr_t)(s->top() + area_touched_words),
79 os::vm_page_size());
80 if (crossing_start != crossing_end) {
81 // If object header crossed a small page boundary we mark the area
82 // as invalid rounding it to a page_size().
83 HeapWord *start = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
84 HeapWord *end = MIN2((HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), page_size()),
85 s->end());
86 invalid = MemRegion(start, end);
87 }
89 ls->add_invalid_region(invalid);
90 }
91 }
92 } else {
93 if (!os::numa_has_static_binding()) {
94 #ifdef ASSERT
95 MemRegion invalid(s->top(), s->end());
96 ls->add_invalid_region(invalid);
97 #else
98 if (ZapUnusedHeapArea) {
99 MemRegion invalid(s->top(), s->end());
100 ls->add_invalid_region(invalid);
101 } else {
102 return;
103 }
104 #endif
105 } else {
106 return;
107 }
108 }
109 }
110 }
112 size_t MutableNUMASpace::used_in_words() const {
113 size_t s = 0;
114 for (int i = 0; i < lgrp_spaces()->length(); i++) {
115 s += lgrp_spaces()->at(i)->space()->used_in_words();
116 }
117 return s;
118 }
120 size_t MutableNUMASpace::free_in_words() const {
121 size_t s = 0;
122 for (int i = 0; i < lgrp_spaces()->length(); i++) {
123 s += lgrp_spaces()->at(i)->space()->free_in_words();
124 }
125 return s;
126 }
129 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
130 guarantee(thr != NULL, "No thread");
131 int lgrp_id = thr->lgrp_id();
132 assert(lgrp_id != -1, "No lgrp_id set");
133 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
134 if (i == -1) {
135 return 0;
136 }
137 return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
138 }
140 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
141 guarantee(thr != NULL, "No thread");
142 int lgrp_id = thr->lgrp_id();
143 assert(lgrp_id != -1, "No lgrp_id set");
144 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
145 if (i == -1) {
146 return 0;
147 }
148 return lgrp_spaces()->at(i)->space()->free_in_bytes();
149 }
151 // Check if the NUMA topology has changed. Add and remove spaces if needed.
152 // The update can be forced by setting the force parameter equal to true.
153 bool MutableNUMASpace::update_layout(bool force) {
154 // Check if the topology had changed.
155 bool changed = os::numa_topology_changed();
156 if (force || changed) {
157 // Compute lgrp intersection. Add/remove spaces.
158 int lgrp_limit = (int)os::numa_get_groups_num();
159 int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit);
160 int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
161 assert(lgrp_num > 0, "There should be at least one locality group");
162 // Add new spaces for the new nodes
163 for (int i = 0; i < lgrp_num; i++) {
164 bool found = false;
165 for (int j = 0; j < lgrp_spaces()->length(); j++) {
166 if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) {
167 found = true;
168 break;
169 }
170 }
171 if (!found) {
172 lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i]));
173 }
174 }
176 // Remove spaces for the removed nodes.
177 for (int i = 0; i < lgrp_spaces()->length();) {
178 bool found = false;
179 for (int j = 0; j < lgrp_num; j++) {
180 if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) {
181 found = true;
182 break;
183 }
184 }
185 if (!found) {
186 delete lgrp_spaces()->at(i);
187 lgrp_spaces()->remove_at(i);
188 } else {
189 i++;
190 }
191 }
193 FREE_C_HEAP_ARRAY(int, lgrp_ids);
195 if (changed) {
196 for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
197 thread->set_lgrp_id(-1);
198 }
199 }
200 return true;
201 }
202 return false;
203 }
205 // Bias region towards the first-touching lgrp. Set the right page sizes.
206 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) {
207 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
208 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
209 if (end > start) {
210 MemRegion aligned_region(start, end);
211 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
212 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
213 assert(region().contains(aligned_region), "Sanity");
214 // First we tell the OS which page size we want in the given range. The underlying
215 // large page can be broken down if we require small pages.
216 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size());
217 // Then we uncommit the pages in the range.
218 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size());
219 // And make them local/first-touch biased.
220 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id);
221 }
222 }
224 // Free all pages in the region.
225 void MutableNUMASpace::free_region(MemRegion mr) {
226 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size());
227 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size());
228 if (end > start) {
229 MemRegion aligned_region(start, end);
230 assert((intptr_t)aligned_region.start() % page_size() == 0 &&
231 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
232 assert(region().contains(aligned_region), "Sanity");
233 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size());
234 }
235 }
237 // Update space layout. Perform adaptation.
238 void MutableNUMASpace::update() {
239 if (update_layout(false)) {
240 // If the topology has changed, make all chunks zero-sized.
241 for (int i = 0; i < lgrp_spaces()->length(); i++) {
242 MutableSpace *s = lgrp_spaces()->at(i)->space();
243 s->set_end(s->bottom());
244 s->set_top(s->bottom());
245 }
246 initialize(region(), true);
247 } else {
248 bool should_initialize = false;
249 if (!os::numa_has_static_binding()) {
250 for (int i = 0; i < lgrp_spaces()->length(); i++) {
251 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) {
252 should_initialize = true;
253 break;
254 }
255 }
256 }
258 if (should_initialize ||
259 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) {
260 initialize(region(), true);
261 }
262 }
264 if (NUMAStats) {
265 for (int i = 0; i < lgrp_spaces()->length(); i++) {
266 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
267 }
268 }
270 scan_pages(NUMAPageScanRate);
271 }
273 // Scan pages. Free pages that have smaller size or wrong placement.
274 void MutableNUMASpace::scan_pages(size_t page_count)
275 {
276 size_t pages_per_chunk = page_count / lgrp_spaces()->length();
277 if (pages_per_chunk > 0) {
278 for (int i = 0; i < lgrp_spaces()->length(); i++) {
279 LGRPSpace *ls = lgrp_spaces()->at(i);
280 ls->scan_pages(page_size(), pages_per_chunk);
281 }
282 }
283 }
285 // Accumulate statistics about the allocation rate of each lgrp.
286 void MutableNUMASpace::accumulate_statistics() {
287 if (UseAdaptiveNUMAChunkSizing) {
288 for (int i = 0; i < lgrp_spaces()->length(); i++) {
289 lgrp_spaces()->at(i)->sample();
290 }
291 increment_samples_count();
292 }
294 if (NUMAStats) {
295 for (int i = 0; i < lgrp_spaces()->length(); i++) {
296 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
297 }
298 }
299 }
301 // Get the current size of a chunk.
302 // This function computes the size of the chunk based on the
303 // difference between chunk ends. This allows it to work correctly in
304 // case the whole space is resized and during the process of adaptive
305 // chunk resizing.
306 size_t MutableNUMASpace::current_chunk_size(int i) {
307 HeapWord *cur_end, *prev_end;
308 if (i == 0) {
309 prev_end = bottom();
310 } else {
311 prev_end = lgrp_spaces()->at(i - 1)->space()->end();
312 }
313 if (i == lgrp_spaces()->length() - 1) {
314 cur_end = end();
315 } else {
316 cur_end = lgrp_spaces()->at(i)->space()->end();
317 }
318 if (cur_end > prev_end) {
319 return pointer_delta(cur_end, prev_end, sizeof(char));
320 }
321 return 0;
322 }
324 // Return the default chunk size by equally diving the space.
325 // page_size() aligned.
326 size_t MutableNUMASpace::default_chunk_size() {
327 return base_space_size() / lgrp_spaces()->length() * page_size();
328 }
330 // Produce a new chunk size. page_size() aligned.
331 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
332 size_t pages_available = base_space_size();
333 for (int j = 0; j < i; j++) {
334 pages_available -= round_down(current_chunk_size(j), page_size()) / page_size();
335 }
336 pages_available -= lgrp_spaces()->length() - i - 1;
337 assert(pages_available > 0, "No pages left");
338 float alloc_rate = 0;
339 for (int j = i; j < lgrp_spaces()->length(); j++) {
340 alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average();
341 }
342 size_t chunk_size = 0;
343 if (alloc_rate > 0) {
344 LGRPSpace *ls = lgrp_spaces()->at(i);
345 chunk_size = (size_t)(ls->alloc_rate()->average() * pages_available / alloc_rate) * page_size();
346 }
347 chunk_size = MAX2(chunk_size, page_size());
349 if (limit > 0) {
350 limit = round_down(limit, page_size());
351 if (chunk_size > current_chunk_size(i)) {
352 chunk_size = MIN2((off_t)chunk_size, (off_t)current_chunk_size(i) + (off_t)limit);
353 } else {
354 chunk_size = MAX2((off_t)chunk_size, (off_t)current_chunk_size(i) - (off_t)limit);
355 }
356 }
357 assert(chunk_size <= pages_available * page_size(), "Chunk size out of range");
358 return chunk_size;
359 }
362 // Return the bottom_region and the top_region. Align them to page_size() boundary.
363 // |------------------new_region---------------------------------|
364 // |----bottom_region--|---intersection---|------top_region------|
365 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
366 MemRegion* bottom_region, MemRegion *top_region) {
367 // Is there bottom?
368 if (new_region.start() < intersection.start()) { // Yes
369 // Try to coalesce small pages into a large one.
370 if (UseLargePages && page_size() >= os::large_page_size()) {
371 HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), os::large_page_size());
372 if (new_region.contains(p)
373 && pointer_delta(p, new_region.start(), sizeof(char)) >= os::large_page_size()) {
374 if (intersection.contains(p)) {
375 intersection = MemRegion(p, intersection.end());
376 } else {
377 intersection = MemRegion(p, p);
378 }
379 }
380 }
381 *bottom_region = MemRegion(new_region.start(), intersection.start());
382 } else {
383 *bottom_region = MemRegion();
384 }
386 // Is there top?
387 if (intersection.end() < new_region.end()) { // Yes
388 // Try to coalesce small pages into a large one.
389 if (UseLargePages && page_size() >= os::large_page_size()) {
390 HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), os::large_page_size());
391 if (new_region.contains(p)
392 && pointer_delta(new_region.end(), p, sizeof(char)) >= os::large_page_size()) {
393 if (intersection.contains(p)) {
394 intersection = MemRegion(intersection.start(), p);
395 } else {
396 intersection = MemRegion(p, p);
397 }
398 }
399 }
400 *top_region = MemRegion(intersection.end(), new_region.end());
401 } else {
402 *top_region = MemRegion();
403 }
404 }
406 // Try to merge the invalid region with the bottom or top region by decreasing
407 // the intersection area. Return the invalid_region aligned to the page_size()
408 // boundary if it's inside the intersection. Return non-empty invalid_region
409 // if it lies inside the intersection (also page-aligned).
410 // |------------------new_region---------------------------------|
411 // |----------------|-------invalid---|--------------------------|
412 // |----bottom_region--|---intersection---|------top_region------|
413 void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection,
414 MemRegion *invalid_region) {
415 if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) {
416 *intersection = MemRegion(invalid_region->end(), intersection->end());
417 *invalid_region = MemRegion();
418 } else
419 if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) {
420 *intersection = MemRegion(intersection->start(), invalid_region->start());
421 *invalid_region = MemRegion();
422 } else
423 if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) {
424 *intersection = MemRegion(new_region.start(), new_region.start());
425 *invalid_region = MemRegion();
426 } else
427 if (intersection->contains(invalid_region)) {
428 // That's the only case we have to make an additional bias_region() call.
429 HeapWord* start = invalid_region->start();
430 HeapWord* end = invalid_region->end();
431 if (UseLargePages && page_size() >= os::large_page_size()) {
432 HeapWord *p = (HeapWord*)round_down((intptr_t) start, os::large_page_size());
433 if (new_region.contains(p)) {
434 start = p;
435 }
436 p = (HeapWord*)round_to((intptr_t) end, os::large_page_size());
437 if (new_region.contains(end)) {
438 end = p;
439 }
440 }
441 if (intersection->start() > start) {
442 *intersection = MemRegion(start, intersection->end());
443 }
444 if (intersection->end() < end) {
445 *intersection = MemRegion(intersection->start(), end);
446 }
447 *invalid_region = MemRegion(start, end);
448 }
449 }
451 void MutableNUMASpace::initialize(MemRegion mr, bool clear_space) {
452 assert(clear_space, "Reallocation will destory data!");
453 assert(lgrp_spaces()->length() > 0, "There should be at least one space");
455 MemRegion old_region = region(), new_region;
456 set_bottom(mr.start());
457 set_end(mr.end());
458 MutableSpace::set_top(bottom());
460 // Compute chunk sizes
461 size_t prev_page_size = page_size();
462 set_page_size(UseLargePages ? os::large_page_size() : os::vm_page_size());
463 HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
464 HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
465 size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
467 // Try small pages if the chunk size is too small
468 if (base_space_size_pages / lgrp_spaces()->length() == 0
469 && page_size() > (size_t)os::vm_page_size()) {
470 set_page_size(os::vm_page_size());
471 rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size());
472 rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size());
473 base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
474 }
475 guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
476 set_base_space_size(base_space_size_pages);
478 // Handle space resize
479 MemRegion top_region, bottom_region;
480 if (!old_region.equals(region())) {
481 new_region = MemRegion(rounded_bottom, rounded_end);
482 MemRegion intersection = new_region.intersection(old_region);
483 if (intersection.start() == NULL ||
484 intersection.end() == NULL ||
485 prev_page_size > page_size()) { // If the page size got smaller we have to change
486 // the page size preference for the whole space.
487 intersection = MemRegion(new_region.start(), new_region.start());
488 }
489 select_tails(new_region, intersection, &bottom_region, &top_region);
490 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id());
491 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id());
492 }
494 // Check if the space layout has changed significantly?
495 // This happens when the space has been resized so that either head or tail
496 // chunk became less than a page.
497 bool layout_valid = UseAdaptiveNUMAChunkSizing &&
498 current_chunk_size(0) > page_size() &&
499 current_chunk_size(lgrp_spaces()->length() - 1) > page_size();
502 for (int i = 0; i < lgrp_spaces()->length(); i++) {
503 LGRPSpace *ls = lgrp_spaces()->at(i);
504 MutableSpace *s = ls->space();
505 old_region = s->region();
507 size_t chunk_byte_size = 0, old_chunk_byte_size = 0;
508 if (i < lgrp_spaces()->length() - 1) {
509 if (!UseAdaptiveNUMAChunkSizing ||
510 (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) ||
511 samples_count() < AdaptiveSizePolicyReadyThreshold) {
512 // No adaptation. Divide the space equally.
513 chunk_byte_size = default_chunk_size();
514 } else
515 if (!layout_valid || NUMASpaceResizeRate == 0) {
516 // Fast adaptation. If no space resize rate is set, resize
517 // the chunks instantly.
518 chunk_byte_size = adaptive_chunk_size(i, 0);
519 } else {
520 // Slow adaptation. Resize the chunks moving no more than
521 // NUMASpaceResizeRate bytes per collection.
522 size_t limit = NUMASpaceResizeRate /
523 (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2);
524 chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size()));
525 }
527 assert(chunk_byte_size >= page_size(), "Chunk size too small");
528 assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check");
529 }
531 if (i == 0) { // Bottom chunk
532 if (i != lgrp_spaces()->length() - 1) {
533 new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
534 } else {
535 new_region = MemRegion(bottom(), end());
536 }
537 } else
538 if (i < lgrp_spaces()->length() - 1) { // Middle chunks
539 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
540 new_region = MemRegion(ps->end(),
541 ps->end() + (chunk_byte_size >> LogHeapWordSize));
542 } else { // Top chunk
543 MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
544 new_region = MemRegion(ps->end(), end());
545 }
546 guarantee(region().contains(new_region), "Region invariant");
549 // The general case:
550 // |---------------------|--invalid---|--------------------------|
551 // |------------------new_region---------------------------------|
552 // |----bottom_region--|---intersection---|------top_region------|
553 // |----old_region----|
554 // The intersection part has all pages in place we don't need to migrate them.
555 // Pages for the top and bottom part should be freed and then reallocated.
557 MemRegion intersection = old_region.intersection(new_region);
559 if (intersection.start() == NULL || intersection.end() == NULL) {
560 intersection = MemRegion(new_region.start(), new_region.start());
561 }
563 if (!os::numa_has_static_binding()) {
564 MemRegion invalid_region = ls->invalid_region().intersection(new_region);
565 // Invalid region is a range of memory that could've possibly
566 // been allocated on the other node. That's relevant only on Solaris where
567 // there is no static memory binding.
568 if (!invalid_region.is_empty()) {
569 merge_regions(new_region, &intersection, &invalid_region);
570 free_region(invalid_region);
571 ls->set_invalid_region(MemRegion());
572 }
573 }
575 select_tails(new_region, intersection, &bottom_region, &top_region);
577 if (!os::numa_has_static_binding()) {
578 // If that's a system with the first-touch policy then it's enough
579 // to free the pages.
580 free_region(bottom_region);
581 free_region(top_region);
582 } else {
583 // In a system with static binding we have to change the bias whenever
584 // we reshape the heap.
585 bias_region(bottom_region, ls->lgrp_id());
586 bias_region(top_region, ls->lgrp_id());
587 }
589 // If we clear the region, we would mangle it in debug. That would cause page
590 // allocation in a different place. Hence setting the top directly.
591 s->initialize(new_region, false);
592 s->set_top(s->bottom());
594 set_adaptation_cycles(samples_count());
595 }
596 }
598 // Set the top of the whole space.
599 // Mark the the holes in chunks below the top() as invalid.
600 void MutableNUMASpace::set_top(HeapWord* value) {
601 bool found_top = false;
602 for (int i = 0; i < lgrp_spaces()->length(); i++) {
603 LGRPSpace *ls = lgrp_spaces()->at(i);
604 MutableSpace *s = ls->space();
605 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
607 if (s->contains(value)) {
608 if (!os::numa_has_static_binding() && top < value && top < s->end()) {
609 ls->add_invalid_region(MemRegion(top, value));
610 }
611 s->set_top(value);
612 found_top = true;
613 } else {
614 if (found_top) {
615 s->set_top(s->bottom());
616 } else {
617 if (!os::numa_has_static_binding() && top < s->end()) {
618 ls->add_invalid_region(MemRegion(top, s->end()));
619 }
620 s->set_top(s->end());
621 }
622 }
623 }
624 MutableSpace::set_top(value);
625 }
627 void MutableNUMASpace::clear() {
628 MutableSpace::set_top(bottom());
629 for (int i = 0; i < lgrp_spaces()->length(); i++) {
630 lgrp_spaces()->at(i)->space()->clear();
631 }
632 }
634 /*
635 Linux supports static memory binding, therefore the most part of the
636 logic dealing with the possible invalid page allocation is effectively
637 disabled. Besides there is no notion of the home node in Linux. A
638 thread is allowed to migrate freely. Although the scheduler is rather
639 reluctant to move threads between the nodes. We check for the current
640 node every allocation. And with a high probability a thread stays on
641 the same node for some time allowing local access to recently allocated
642 objects.
643 */
645 HeapWord* MutableNUMASpace::allocate(size_t size) {
646 Thread* thr = Thread::current();
647 int lgrp_id = thr->lgrp_id();
648 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
649 lgrp_id = os::numa_get_group_id();
650 thr->set_lgrp_id(lgrp_id);
651 }
653 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
655 // It is possible that a new CPU has been hotplugged and
656 // we haven't reshaped the space accordingly.
657 if (i == -1) {
658 i = os::random() % lgrp_spaces()->length();
659 }
661 MutableSpace *s = lgrp_spaces()->at(i)->space();
662 HeapWord *p = s->allocate(size);
664 if (p != NULL) {
665 size_t remainder = s->free_in_words();
666 if (remainder < (size_t)oopDesc::header_size() && remainder > 0) {
667 s->set_top(s->top() - size);
668 p = NULL;
669 }
670 }
671 if (p != NULL) {
672 if (top() < s->top()) { // Keep _top updated.
673 MutableSpace::set_top(s->top());
674 }
675 }
676 // Make the page allocation happen here if there is no static binding..
677 if (p != NULL && !os::numa_has_static_binding()) {
678 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
679 *(int*)i = 0;
680 }
681 }
682 return p;
683 }
685 // This version is lock-free.
686 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
687 Thread* thr = Thread::current();
688 int lgrp_id = thr->lgrp_id();
689 if (lgrp_id == -1 || !os::numa_has_group_homing()) {
690 lgrp_id = os::numa_get_group_id();
691 thr->set_lgrp_id(lgrp_id);
692 }
694 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
695 // It is possible that a new CPU has been hotplugged and
696 // we haven't reshaped the space accordingly.
697 if (i == -1) {
698 i = os::random() % lgrp_spaces()->length();
699 }
700 MutableSpace *s = lgrp_spaces()->at(i)->space();
701 HeapWord *p = s->cas_allocate(size);
702 if (p != NULL) {
703 size_t remainder = pointer_delta(s->end(), p);
704 if (remainder < (size_t)oopDesc::header_size() && remainder > 0) {
705 if (s->cas_deallocate(p, size)) {
706 // We were the last to allocate and created a fragment less than
707 // a minimal object.
708 p = NULL;
709 }
710 }
711 }
712 if (p != NULL) {
713 HeapWord* cur_top, *cur_chunk_top = p + size;
714 while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated.
715 if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) {
716 break;
717 }
718 }
719 }
721 // Make the page allocation happen here if there is no static binding.
722 if (p != NULL && !os::numa_has_static_binding() ) {
723 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
724 *(int*)i = 0;
725 }
726 }
727 return p;
728 }
730 void MutableNUMASpace::print_short_on(outputStream* st) const {
731 MutableSpace::print_short_on(st);
732 st->print(" (");
733 for (int i = 0; i < lgrp_spaces()->length(); i++) {
734 st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id());
735 lgrp_spaces()->at(i)->space()->print_short_on(st);
736 if (i < lgrp_spaces()->length() - 1) {
737 st->print(", ");
738 }
739 }
740 st->print(")");
741 }
743 void MutableNUMASpace::print_on(outputStream* st) const {
744 MutableSpace::print_on(st);
745 for (int i = 0; i < lgrp_spaces()->length(); i++) {
746 LGRPSpace *ls = lgrp_spaces()->at(i);
747 st->print(" lgrp %d", ls->lgrp_id());
748 ls->space()->print_on(st);
749 if (NUMAStats) {
750 for (int i = 0; i < lgrp_spaces()->length(); i++) {
751 lgrp_spaces()->at(i)->accumulate_statistics(page_size());
752 }
753 st->print(" local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n",
754 ls->space_stats()->_local_space / K,
755 ls->space_stats()->_remote_space / K,
756 ls->space_stats()->_unbiased_space / K,
757 ls->space_stats()->_uncommited_space / K,
758 ls->space_stats()->_large_pages,
759 ls->space_stats()->_small_pages);
760 }
761 }
762 }
764 void MutableNUMASpace::verify(bool allow_dirty) const {
765 for (int i = 0; i < lgrp_spaces()->length(); i++) {
766 lgrp_spaces()->at(i)->space()->verify(allow_dirty);
767 }
768 }
770 // Scan pages and gather stats about page placement and size.
771 void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) {
772 clear_space_stats();
773 char *start = (char*)round_to((intptr_t) space()->bottom(), page_size);
774 char* end = (char*)round_down((intptr_t) space()->end(), page_size);
775 if (start < end) {
776 for (char *p = start; p < end;) {
777 os::page_info info;
778 if (os::get_page_info(p, &info)) {
779 if (info.size > 0) {
780 if (info.size > (size_t)os::vm_page_size()) {
781 space_stats()->_large_pages++;
782 } else {
783 space_stats()->_small_pages++;
784 }
785 if (info.lgrp_id == lgrp_id()) {
786 space_stats()->_local_space += info.size;
787 } else {
788 space_stats()->_remote_space += info.size;
789 }
790 p += info.size;
791 } else {
792 p += os::vm_page_size();
793 space_stats()->_uncommited_space += os::vm_page_size();
794 }
795 } else {
796 return;
797 }
798 }
799 }
800 space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) +
801 pointer_delta(space()->end(), end, sizeof(char));
803 }
805 // Scan page_count pages and verify if they have the right size and right placement.
806 // If invalid pages are found they are freed in hope that subsequent reallocation
807 // will be more successful.
808 void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count)
809 {
810 char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size);
811 char* range_end = (char*)round_down((intptr_t) space()->end(), page_size);
813 if (range_start > last_page_scanned() || last_page_scanned() >= range_end) {
814 set_last_page_scanned(range_start);
815 }
817 char *scan_start = last_page_scanned();
818 char* scan_end = MIN2(scan_start + page_size * page_count, range_end);
820 os::page_info page_expected, page_found;
821 page_expected.size = page_size;
822 page_expected.lgrp_id = lgrp_id();
824 char *s = scan_start;
825 while (s < scan_end) {
826 char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found);
827 if (e == NULL) {
828 break;
829 }
830 if (e != scan_end) {
831 if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id())
832 && page_expected.size != 0) {
833 os::free_memory(s, pointer_delta(e, s, sizeof(char)));
834 }
835 page_expected = page_found;
836 }
837 s = e;
838 }
840 set_last_page_scanned(scan_end);
841 }