diff -r f90c822e73f8 -r 2d8a650513c2 src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp --- a/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp Wed Apr 27 01:25:04 2016 +0800 +++ b/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp Fri Apr 29 00:06:10 2016 +0800 @@ -22,12 +22,19 @@ * */ +/* + * This file has been modified by Loongson Technology in 2015. These + * modifications are Copyright (c) 2015 Loongson Technology, and are made + * available on the same license terms set forth above. + */ + #ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_INLINE_HPP #define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_INLINE_HPP #include "gc_implementation/parallelScavenge/psOldGen.hpp" #include "gc_implementation/parallelScavenge/psPromotionManager.hpp" #include "gc_implementation/parallelScavenge/psScavenge.hpp" +#include "gc_implementation/shared/mutableNUMASpace.hpp" #include "oops/oop.psgc.inline.hpp" inline PSPromotionManager* PSPromotionManager::manager_array(int index) { @@ -68,6 +75,11 @@ // into smaller submethods, but we need to be careful not to hurt // performance. // + +extern int node_ex; +extern int each_gc_copy_fre[16]; +extern float each_gc_copy_time[16]; + template oop PSPromotionManager::copy_to_survivor_space(oop o) { assert(PSScavenge::should_scavenge(&o), "Sanity"); @@ -83,6 +95,10 @@ if (!test_mark->is_marked()) { bool new_obj_is_tenured = false; size_t new_obj_size = o->size(); + + if(UseStasticScavenge) { + stastic_scavenge(o); + } if (!promote_immediately) { // Find the objects age, MT safe. @@ -122,53 +138,154 @@ } #endif // #ifndef PRODUCT - new_obj = (oop) _old_lab.allocate(new_obj_size); - new_obj_is_tenured = true; + if(UseOldNUMA) { +/* 2014/7/7 Liao: Copy objects to the same node of current GC thread */ + if(UseNUMAGC) { + new_obj = (oop) _old_lab_oldnuma[os::numa_get_group_id()].allocate(new_obj_size); + new_obj_is_tenured = true; - if (new_obj == NULL) { - if (!_old_gen_is_full) { - // Do we allocate directly, or flush and refill? - if (new_obj_size > (OldPLABSize / 2)) { - // Allocate this object directly - new_obj = (oop)old_gen()->cas_allocate(new_obj_size); - } else { - // Flush and fill - _old_lab.flush(); + if (new_obj == NULL) { + if (!_old_gen_is_full) { + // Do we allocate directly, or flush and refill? + if (new_obj_size > (OldPLABSize / 2)) { + // Allocate this object directly + new_obj = (oop)old_gen()->cas_allocate(new_obj_size, os::numa_get_group_id()); + } else { + // Flush and fill + _old_lab_oldnuma[os::numa_get_group_id()].flush(); - HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize); - if(lab_base != NULL) { -#ifdef ASSERT - // Delay the initialization of the promotion lab (plab). - // This exposes uninitialized plabs to card table processing. - if (GCWorkerDelayMillis > 0) { - os::sleep(Thread::current(), GCWorkerDelayMillis, false); + HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize, os::numa_get_group_id()); + if(lab_base != NULL) { + _old_lab_oldnuma[os::numa_get_group_id()].initialize(MemRegion(lab_base, OldPLABSize)); + // Try the old lab allocation again. + new_obj = (oop) _old_lab_oldnuma[os::numa_get_group_id()].allocate(new_obj_size); + } } -#endif - _old_lab.initialize(MemRegion(lab_base, OldPLABSize)); - // Try the old lab allocation again. - new_obj = (oop) _old_lab.allocate(new_obj_size); + } + + // This is the promotion failed test, and code handling. + // The code belongs here for two reasons. It is slightly + // different than the code below, and cannot share the + // CAS testing code. Keeping the code here also minimizes + // the impact on the common case fast path code. + + if (new_obj == NULL) { + _old_gen_is_full = true; + return oop_promotion_failed(o, test_mark); } } } + else { + ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap(); + MutableNUMASpace* s = (MutableNUMASpace*) heap->old_gen()->object_space(); + int i = s->lgrp_spaces()->length(); + int node; + if(i > 1) { + node = node_ex % (i - 1) + 1; + node_ex++; + } + else + node = 0; - // This is the promotion failed test, and code handling. - // The code belongs here for two reasons. It is slightly - // different than the code below, and cannot share the - // CAS testing code. Keeping the code here also minimizes - // the impact on the common case fast path code. + new_obj = (oop) _old_lab_oldnuma[node].allocate(new_obj_size); + new_obj_is_tenured = true; + + if (new_obj == NULL) { + if (!_old_gen_is_full) { + // Do we allocate directly, or flush and refill? + if (new_obj_size > (OldPLABSize / 2)) { + // Allocate this object directly + new_obj = (oop)old_gen()->cas_allocate(new_obj_size, node); + } else { + // Flush and fill + _old_lab_oldnuma[node].flush(); + + HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize, node); + if(lab_base != NULL) { + _old_lab_oldnuma[node].initialize(MemRegion(lab_base, OldPLABSize)); + // Try the old lab allocation again. + new_obj = (oop) _old_lab_oldnuma[node].allocate(new_obj_size); + } + } + } + + // This is the promotion failed test, and code handling. + // The code belongs here for two reasons. It is slightly + // different than the code below, and cannot share the + // CAS testing code. Keeping the code here also minimizes + // the impact on the common case fast path code. + + if (new_obj == NULL) { + _old_gen_is_full = true; + return oop_promotion_failed(o, test_mark); + } + } + } + } + else { + new_obj = (oop) _old_lab.allocate(new_obj_size); + new_obj_is_tenured = true; if (new_obj == NULL) { - _old_gen_is_full = true; - return oop_promotion_failed(o, test_mark); + if (!_old_gen_is_full) { + // Do we allocate directly, or flush and refill? + if (new_obj_size > (OldPLABSize / 2)) { + // Allocate this object directly + new_obj = (oop)old_gen()->cas_allocate(new_obj_size, 0); + } else { + // Flush and fill + _old_lab.flush(); + + HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize, 0); + if(lab_base != NULL) { +#ifdef ASSERT + // Delay the initialization of the promotion lab (plab). + // This exposes uninitialized plabs to card table processing. + if (GCWorkerDelayMillis > 0) { + os::sleep(Thread::current(), GCWorkerDelayMillis, false); + } +#endif + _old_lab.initialize(MemRegion(lab_base, OldPLABSize)); + // Try the old lab allocation again. + new_obj = (oop) _old_lab.allocate(new_obj_size); + } + } + } + + // This is the promotion failed test, and code handling. + // The code belongs here for two reasons. It is slightly + // different than the code below, and cannot share the + // CAS testing code. Keeping the code here also minimizes + // the impact on the common case fast path code. + + if (new_obj == NULL) { + _old_gen_is_full = true; + return oop_promotion_failed(o, test_mark); + } } } } assert(new_obj != NULL, "allocation should have succeeded"); + TimeStamp before_copy, after_copy; + + if(UseStasticCopy) { + before_copy.update(); + } + // Copy obj Copy::aligned_disjoint_words((HeapWord*)o, (HeapWord*)new_obj, new_obj_size); + if(UseStasticCopy) { + after_copy.update(); + } + + if(UseStasticCopy) { + each_gc_copy_time[os::numa_get_cpu_id()] += after_copy.ticks() - before_copy.ticks(); + each_gc_copy_fre[os::numa_get_cpu_id()]++; + } + // Now we have to CAS in the header. if (o->cas_forward_to(new_obj, test_mark)) { // We won any races, we "own" this object. @@ -205,8 +322,20 @@ // deallocate it, so we have to test. If the deallocation fails, // overwrite with a filler object. if (new_obj_is_tenured) { - if (!_old_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) { - CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size); + if(UseOldNUMA) { + ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap(); + MutableNUMASpace* s = (MutableNUMASpace*) heap->old_gen()->object_space(); + int i; + for(i = 0; i < s->lgrp_spaces()->length(); i++) { + if (!_old_lab_oldnuma[i].unallocate_object((HeapWord*) new_obj, new_obj_size)) { + CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size); + } + } + } + else { + if (!_old_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) { + CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size); + } } } else if (!_young_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) { CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size); @@ -233,7 +362,6 @@ return new_obj; } - inline void PSPromotionManager::process_popped_location_depth(StarTask p) { if (is_oop_masked(p)) { assert(PSChunkLargeArrays, "invariant");