Mercurial > jdk8-mips64-public > hotspot / changeset
changeset
Merge
Tue, 17 Jan 2012 21:25:28 -0500
- author
- kamg
- date
- Tue, 17 Jan 2012 21:25:28 -0500
- changeset 3431
- 6520f9861937
- parent 3430
- d7e3846464d0
- parent 3429
- f1cd52d6ce02
- child 3432
- db18ca98d237
Merge
1.1 --- a/.hgtags Tue Jan 17 13:08:52 2012 -0500 1.2 +++ b/.hgtags Tue Jan 17 21:25:28 2012 -0500 1.3 @@ -207,3 +207,7 @@ 1.4 a2fef924d8e6f37dac2a887315e3502876cc8e24 hs23-b08 1.5 61165f53f1656b9f99e4fb806429bf98b99d59c3 jdk8-b18 1.6 4bcf61041217f8677dcec18e90e9196acc945bba hs23-b09 1.7 +9232e0ecbc2cec54dcc8f93004fb00c214446460 jdk8-b19 1.8 +fe2c8764998112b7fefcd7d41599714813ae4327 jdk8-b20 1.9 +9952d1c439d64c5fd4ad1236a63a62bd5a49d4c3 jdk8-b21 1.10 +513351373923f74a7c91755748b95c9771e59f96 hs23-b10
2.1 --- a/agent/src/share/classes/sun/jvm/hotspot/oops/InstanceKlass.java Tue Jan 17 13:08:52 2012 -0500 2.2 +++ b/agent/src/share/classes/sun/jvm/hotspot/oops/InstanceKlass.java Tue Jan 17 21:25:28 2012 -0500 2.3 @@ -63,8 +63,6 @@ 2.4 private static int CLASS_STATE_FULLY_INITIALIZED; 2.5 private static int CLASS_STATE_INITIALIZATION_ERROR; 2.6 2.7 - private static int IS_MARKED_DEPENDENT_MASK; 2.8 - 2.9 private static synchronized void initialize(TypeDataBase db) throws WrongTypeException { 2.10 Type type = db.lookupType("instanceKlass"); 2.11 arrayKlasses = new OopField(type.getOopField("_array_klasses"), Oop.getHeaderSize()); 2.12 @@ -92,7 +90,7 @@ 2.13 staticFieldSize = new CIntField(type.getCIntegerField("_static_field_size"), Oop.getHeaderSize()); 2.14 staticOopFieldCount = new CIntField(type.getCIntegerField("_static_oop_field_count"), Oop.getHeaderSize()); 2.15 nonstaticOopMapSize = new CIntField(type.getCIntegerField("_nonstatic_oop_map_size"), Oop.getHeaderSize()); 2.16 - miscFlags = new CIntField(type.getCIntegerField("_misc_flags"), Oop.getHeaderSize()); 2.17 + isMarkedDependent = new CIntField(type.getCIntegerField("_is_marked_dependent"), Oop.getHeaderSize()); 2.18 initState = new CIntField(type.getCIntegerField("_init_state"), Oop.getHeaderSize()); 2.19 vtableLen = new CIntField(type.getCIntegerField("_vtable_len"), Oop.getHeaderSize()); 2.20 itableLen = new CIntField(type.getCIntegerField("_itable_len"), Oop.getHeaderSize()); 2.21 @@ -120,8 +118,6 @@ 2.22 CLASS_STATE_FULLY_INITIALIZED = db.lookupIntConstant("instanceKlass::fully_initialized").intValue(); 2.23 CLASS_STATE_INITIALIZATION_ERROR = db.lookupIntConstant("instanceKlass::initialization_error").intValue(); 2.24 2.25 - IS_MARKED_DEPENDENT_MASK = db.lookupIntConstant("instanceKlass::IS_MARKED_DEPENDENT").intValue(); 2.26 - 2.27 } 2.28 2.29 InstanceKlass(OopHandle handle, ObjectHeap heap) { 2.30 @@ -155,7 +151,7 @@ 2.31 private static CIntField staticFieldSize; 2.32 private static CIntField staticOopFieldCount; 2.33 private static CIntField nonstaticOopMapSize; 2.34 - private static CIntField miscFlags; 2.35 + private static CIntField isMarkedDependent; 2.36 private static CIntField initState; 2.37 private static CIntField vtableLen; 2.38 private static CIntField itableLen; 2.39 @@ -337,7 +333,7 @@ 2.40 public long getNonstaticFieldSize() { return nonstaticFieldSize.getValue(this); } 2.41 public long getStaticOopFieldCount() { return staticOopFieldCount.getValue(this); } 2.42 public long getNonstaticOopMapSize() { return nonstaticOopMapSize.getValue(this); } 2.43 - public boolean getIsMarkedDependent() { return (miscFlags.getValue(this) & IS_MARKED_DEPENDENT_MASK) != 0; } 2.44 + public boolean getIsMarkedDependent() { return isMarkedDependent.getValue(this) != 0; } 2.45 public long getVtableLen() { return vtableLen.getValue(this); } 2.46 public long getItableLen() { return itableLen.getValue(this); } 2.47 public Symbol getGenericSignature() { return getSymbol(genericSignature); } 2.48 @@ -528,7 +524,7 @@ 2.49 visitor.doCInt(staticFieldSize, true); 2.50 visitor.doCInt(staticOopFieldCount, true); 2.51 visitor.doCInt(nonstaticOopMapSize, true); 2.52 - visitor.doCInt(miscFlags, true); 2.53 + visitor.doCInt(isMarkedDependent, true); 2.54 visitor.doCInt(initState, true); 2.55 visitor.doCInt(vtableLen, true); 2.56 visitor.doCInt(itableLen, true);
3.1 --- a/make/hotspot_version Tue Jan 17 13:08:52 2012 -0500 3.2 +++ b/make/hotspot_version Tue Jan 17 21:25:28 2012 -0500 3.3 @@ -35,7 +35,7 @@ 3.4 3.5 HS_MAJOR_VER=23 3.6 HS_MINOR_VER=0 3.7 -HS_BUILD_NUMBER=10 3.8 +HS_BUILD_NUMBER=11 3.9 3.10 JDK_MAJOR_VER=1 3.11 JDK_MINOR_VER=8
4.1 --- a/src/cpu/sparc/vm/sparc.ad Tue Jan 17 13:08:52 2012 -0500 4.2 +++ b/src/cpu/sparc/vm/sparc.ad Tue Jan 17 21:25:28 2012 -0500 4.3 @@ -9283,6 +9283,7 @@ 4.4 // (compare 'operand indIndex' and 'instruct addP_reg_reg' above) 4.5 instruct jumpXtnd(iRegX switch_val, o7RegI table) %{ 4.6 match(Jump switch_val); 4.7 + effect(TEMP table); 4.8 4.9 ins_cost(350); 4.10 4.11 @@ -10273,24 +10274,24 @@ 4.12 // ============================================================================ 4.13 // inlined locking and unlocking 4.14 4.15 -instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, o7RegP scratch ) %{ 4.16 +instruct cmpFastLock(flagsRegP pcc, iRegP object, o1RegP box, iRegP scratch2, o7RegP scratch ) %{ 4.17 match(Set pcc (FastLock object box)); 4.18 4.19 - effect(KILL scratch, TEMP scratch2); 4.20 + effect(TEMP scratch2, USE_KILL box, KILL scratch); 4.21 ins_cost(100); 4.22 4.23 - format %{ "FASTLOCK $object, $box; KILL $scratch, $scratch2, $box" %} 4.24 + format %{ "FASTLOCK $object,$box\t! kills $box,$scratch,$scratch2" %} 4.25 ins_encode( Fast_Lock(object, box, scratch, scratch2) ); 4.26 ins_pipe(long_memory_op); 4.27 %} 4.28 4.29 4.30 -instruct cmpFastUnlock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, o7RegP scratch ) %{ 4.31 +instruct cmpFastUnlock(flagsRegP pcc, iRegP object, o1RegP box, iRegP scratch2, o7RegP scratch ) %{ 4.32 match(Set pcc (FastUnlock object box)); 4.33 - effect(KILL scratch, TEMP scratch2); 4.34 + effect(TEMP scratch2, USE_KILL box, KILL scratch); 4.35 ins_cost(100); 4.36 4.37 - format %{ "FASTUNLOCK $object, $box; KILL $scratch, $scratch2, $box" %} 4.38 + format %{ "FASTUNLOCK $object,$box\t! kills $box,$scratch,$scratch2" %} 4.39 ins_encode( Fast_Unlock(object, box, scratch, scratch2) ); 4.40 ins_pipe(long_memory_op); 4.41 %}
5.1 --- a/src/cpu/x86/vm/x86_32.ad Tue Jan 17 13:08:52 2012 -0500 5.2 +++ b/src/cpu/x86/vm/x86_32.ad Tue Jan 17 21:25:28 2012 -0500 5.3 @@ -13435,20 +13435,20 @@ 5.4 // inlined locking and unlocking 5.5 5.6 5.7 -instruct cmpFastLock( eFlagsReg cr, eRegP object, eRegP box, eAXRegI tmp, eRegP scr) %{ 5.8 +instruct cmpFastLock( eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eRegP scr) %{ 5.9 match( Set cr (FastLock object box) ); 5.10 - effect( TEMP tmp, TEMP scr ); 5.11 + effect( TEMP tmp, TEMP scr, USE_KILL box ); 5.12 ins_cost(300); 5.13 - format %{ "FASTLOCK $object, $box KILLS $tmp,$scr" %} 5.14 + format %{ "FASTLOCK $object,$box\t! kills $box,$tmp,$scr" %} 5.15 ins_encode( Fast_Lock(object,box,tmp,scr) ); 5.16 ins_pipe( pipe_slow ); 5.17 %} 5.18 5.19 instruct cmpFastUnlock( eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{ 5.20 match( Set cr (FastUnlock object box) ); 5.21 - effect( TEMP tmp ); 5.22 + effect( TEMP tmp, USE_KILL box ); 5.23 ins_cost(300); 5.24 - format %{ "FASTUNLOCK $object, $box, $tmp" %} 5.25 + format %{ "FASTUNLOCK $object,$box\t! kills $box,$tmp" %} 5.26 ins_encode( Fast_Unlock(object,box,tmp) ); 5.27 ins_pipe( pipe_slow ); 5.28 %}
6.1 --- a/src/cpu/x86/vm/x86_64.ad Tue Jan 17 13:08:52 2012 -0500 6.2 +++ b/src/cpu/x86/vm/x86_64.ad Tue Jan 17 21:25:28 2012 -0500 6.3 @@ -11511,13 +11511,13 @@ 6.4 // inlined locking and unlocking 6.5 6.6 instruct cmpFastLock(rFlagsReg cr, 6.7 - rRegP object, rRegP box, rax_RegI tmp, rRegP scr) 6.8 + rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) 6.9 %{ 6.10 match(Set cr (FastLock object box)); 6.11 - effect(TEMP tmp, TEMP scr); 6.12 + effect(TEMP tmp, TEMP scr, USE_KILL box); 6.13 6.14 ins_cost(300); 6.15 - format %{ "fastlock $object,$box,$tmp,$scr" %} 6.16 + format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %} 6.17 ins_encode(Fast_Lock(object, box, tmp, scr)); 6.18 ins_pipe(pipe_slow); 6.19 %} 6.20 @@ -11526,10 +11526,10 @@ 6.21 rRegP object, rax_RegP box, rRegP tmp) 6.22 %{ 6.23 match(Set cr (FastUnlock object box)); 6.24 - effect(TEMP tmp); 6.25 + effect(TEMP tmp, USE_KILL box); 6.26 6.27 ins_cost(300); 6.28 - format %{ "fastunlock $object, $box, $tmp" %} 6.29 + format %{ "fastunlock $object,$box\t! kills $box,$tmp" %} 6.30 ins_encode(Fast_Unlock(object, box, tmp)); 6.31 ins_pipe(pipe_slow); 6.32 %}
7.1 --- a/src/share/vm/ci/ciTypeFlow.cpp Tue Jan 17 13:08:52 2012 -0500 7.2 +++ b/src/share/vm/ci/ciTypeFlow.cpp Tue Jan 17 21:25:28 2012 -0500 7.3 @@ -1589,7 +1589,7 @@ 7.4 _next = NULL; 7.5 _on_work_list = false; 7.6 _backedge_copy = false; 7.7 - _exception_entry = false; 7.8 + _has_monitorenter = false; 7.9 _trap_bci = -1; 7.10 _trap_index = 0; 7.11 df_init(); 7.12 @@ -2182,6 +2182,10 @@ 7.13 !head->is_clonable_exit(lp)) 7.14 continue; 7.15 7.16 + // Avoid BoxLock merge. 7.17 + if (EliminateNestedLocks && head->has_monitorenter()) 7.18 + continue; 7.19 + 7.20 // check not already cloned 7.21 if (head->backedge_copy_count() != 0) 7.22 continue; 7.23 @@ -2322,6 +2326,10 @@ 7.24 // Watch for bailouts. 7.25 if (failing()) return; 7.26 7.27 + if (str.cur_bc() == Bytecodes::_monitorenter) { 7.28 + block->set_has_monitorenter(); 7.29 + } 7.30 + 7.31 if (res) { 7.32 7.33 // We have encountered a trap. Record it in this block.
8.1 --- a/src/share/vm/ci/ciTypeFlow.hpp Tue Jan 17 13:08:52 2012 -0500 8.2 +++ b/src/share/vm/ci/ciTypeFlow.hpp Tue Jan 17 21:25:28 2012 -0500 8.3 @@ -544,15 +544,19 @@ 8.4 // Has this block been cloned for a loop backedge? 8.5 bool _backedge_copy; 8.6 8.7 + // This block is entry to irreducible loop. 8.8 + bool _irreducible_entry; 8.9 + 8.10 + // This block has monitor entry point. 8.11 + bool _has_monitorenter; 8.12 + 8.13 // A pointer used for our internal work list 8.14 + bool _on_work_list; // on the work list 8.15 Block* _next; 8.16 - bool _on_work_list; // on the work list 8.17 Block* _rpo_next; // Reverse post order list 8.18 8.19 // Loop info 8.20 Loop* _loop; // nearest loop 8.21 - bool _irreducible_entry; // entry to irreducible loop 8.22 - bool _exception_entry; // entry to exception handler 8.23 8.24 ciBlock* ciblock() const { return _ciblock; } 8.25 StateVector* state() const { return _state; } 8.26 @@ -689,6 +693,8 @@ 8.27 bool is_loop_head() const { return _loop && _loop->head() == this; } 8.28 void set_irreducible_entry(bool c) { _irreducible_entry = c; } 8.29 bool is_irreducible_entry() const { return _irreducible_entry; } 8.30 + void set_has_monitorenter() { _has_monitorenter = true; } 8.31 + bool has_monitorenter() const { return _has_monitorenter; } 8.32 bool is_visited() const { return has_pre_order(); } 8.33 bool is_post_visited() const { return has_post_order(); } 8.34 bool is_clonable_exit(Loop* lp);
9.1 --- a/src/share/vm/code/dependencies.cpp Tue Jan 17 13:08:52 2012 -0500 9.2 +++ b/src/share/vm/code/dependencies.cpp Tue Jan 17 21:25:28 2012 -0500 9.3 @@ -1631,7 +1631,7 @@ 9.4 for (ContextStream str(*this); str.next(); ) { 9.5 klassOop d = str.klass(); 9.6 assert(!instanceKlass::cast(d)->is_marked_dependent(), "checking"); 9.7 - instanceKlass::cast(d)->set_is_marked_dependent(); 9.8 + instanceKlass::cast(d)->set_is_marked_dependent(true); 9.9 } 9.10 } 9.11 9.12 @@ -1640,7 +1640,7 @@ 9.13 // Unmark transitive interfaces 9.14 for (ContextStream str(*this); str.next(); ) { 9.15 klassOop d = str.klass(); 9.16 - instanceKlass::cast(d)->clear_is_marked_dependent(); 9.17 + instanceKlass::cast(d)->set_is_marked_dependent(false); 9.18 } 9.19 } 9.20
10.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp Tue Jan 17 13:08:52 2012 -0500 10.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp Tue Jan 17 21:25:28 2012 -0500 10.3 @@ -1,5 +1,5 @@ 10.4 /* 10.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 10.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 10.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 10.8 * 10.9 * This code is free software; you can redistribute it and/or modify it 10.10 @@ -31,6 +31,7 @@ 10.11 #include "gc_implementation/g1/g1ErgoVerbose.hpp" 10.12 #include "gc_implementation/g1/g1OopClosures.inline.hpp" 10.13 #include "gc_implementation/g1/g1RemSet.hpp" 10.14 +#include "gc_implementation/g1/heapRegion.inline.hpp" 10.15 #include "gc_implementation/g1/heapRegionRemSet.hpp" 10.16 #include "gc_implementation/g1/heapRegionSeq.inline.hpp" 10.17 #include "gc_implementation/shared/vmGCOperations.hpp" 10.18 @@ -183,12 +184,11 @@ 10.19 void CMMarkStack::allocate(size_t size) { 10.20 _base = NEW_C_HEAP_ARRAY(oop, size); 10.21 if (_base == NULL) { 10.22 - vm_exit_during_initialization("Failed to allocate " 10.23 - "CM region mark stack"); 10.24 + vm_exit_during_initialization("Failed to allocate CM region mark stack"); 10.25 } 10.26 _index = 0; 10.27 _capacity = (jint) size; 10.28 - _oops_do_bound = -1; 10.29 + _saved_index = -1; 10.30 NOT_PRODUCT(_max_depth = 0); 10.31 } 10.32 10.33 @@ -283,7 +283,6 @@ 10.34 } 10.35 } 10.36 10.37 - 10.38 CMRegionStack::CMRegionStack() : _base(NULL) {} 10.39 10.40 void CMRegionStack::allocate(size_t size) { 10.41 @@ -302,6 +301,8 @@ 10.42 } 10.43 10.44 void CMRegionStack::push_lock_free(MemRegion mr) { 10.45 + guarantee(false, "push_lock_free(): don't call this any more"); 10.46 + 10.47 assert(mr.word_size() > 0, "Precondition"); 10.48 while (true) { 10.49 jint index = _index; 10.50 @@ -325,6 +326,8 @@ 10.51 // marking / remark phases. Should only be called in tandem with 10.52 // other lock-free pops. 10.53 MemRegion CMRegionStack::pop_lock_free() { 10.54 + guarantee(false, "pop_lock_free(): don't call this any more"); 10.55 + 10.56 while (true) { 10.57 jint index = _index; 10.58 10.59 @@ -390,6 +393,8 @@ 10.60 #endif 10.61 10.62 bool CMRegionStack::invalidate_entries_into_cset() { 10.63 + guarantee(false, "invalidate_entries_into_cset(): don't call this any more"); 10.64 + 10.65 bool result = false; 10.66 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 10.67 for (int i = 0; i < _oops_do_bound; ++i) { 10.68 @@ -438,14 +443,29 @@ 10.69 return res; 10.70 } 10.71 10.72 +void CMMarkStack::note_start_of_gc() { 10.73 + assert(_saved_index == -1, 10.74 + "note_start_of_gc()/end_of_gc() bracketed incorrectly"); 10.75 + _saved_index = _index; 10.76 +} 10.77 + 10.78 +void CMMarkStack::note_end_of_gc() { 10.79 + // This is intentionally a guarantee, instead of an assert. If we 10.80 + // accidentally add something to the mark stack during GC, it 10.81 + // will be a correctness issue so it's better if we crash. we'll 10.82 + // only check this once per GC anyway, so it won't be a performance 10.83 + // issue in any way. 10.84 + guarantee(_saved_index == _index, 10.85 + err_msg("saved index: %d index: %d", _saved_index, _index)); 10.86 + _saved_index = -1; 10.87 +} 10.88 + 10.89 void CMMarkStack::oops_do(OopClosure* f) { 10.90 - if (_index == 0) return; 10.91 - assert(_oops_do_bound != -1 && _oops_do_bound <= _index, 10.92 - "Bound must be set."); 10.93 - for (int i = 0; i < _oops_do_bound; i++) { 10.94 + assert(_saved_index == _index, 10.95 + err_msg("saved index: %d index: %d", _saved_index, _index)); 10.96 + for (int i = 0; i < _index; i += 1) { 10.97 f->do_oop(&_base[i]); 10.98 } 10.99 - _oops_do_bound = -1; 10.100 } 10.101 10.102 bool ConcurrentMark::not_yet_marked(oop obj) const { 10.103 @@ -783,7 +803,7 @@ 10.104 public: 10.105 bool doHeapRegion(HeapRegion* r) { 10.106 if (!r->continuesHumongous()) { 10.107 - r->note_start_of_marking(true); 10.108 + r->note_start_of_marking(); 10.109 } 10.110 return false; 10.111 } 10.112 @@ -804,6 +824,10 @@ 10.113 10.114 // Initialise marking structures. This has to be done in a STW phase. 10.115 reset(); 10.116 + 10.117 + // For each region note start of marking. 10.118 + NoteStartOfMarkHRClosure startcl; 10.119 + g1h->heap_region_iterate(&startcl); 10.120 } 10.121 10.122 10.123 @@ -818,10 +842,6 @@ 10.124 // every remark and we'll eventually not need to cause one. 10.125 force_overflow_stw()->init(); 10.126 10.127 - // For each region note start of marking. 10.128 - NoteStartOfMarkHRClosure startcl; 10.129 - g1h->heap_region_iterate(&startcl); 10.130 - 10.131 // Start Concurrent Marking weak-reference discovery. 10.132 ReferenceProcessor* rp = g1h->ref_processor_cm(); 10.133 // enable ("weak") refs discovery 10.134 @@ -946,22 +966,9 @@ 10.135 } 10.136 #endif // !PRODUCT 10.137 10.138 -void ConcurrentMark::grayRoot(oop p) { 10.139 - HeapWord* addr = (HeapWord*) p; 10.140 - // We can't really check against _heap_start and _heap_end, since it 10.141 - // is possible during an evacuation pause with piggy-backed 10.142 - // initial-mark that the committed space is expanded during the 10.143 - // pause without CM observing this change. So the assertions below 10.144 - // is a bit conservative; but better than nothing. 10.145 - assert(_g1h->g1_committed().contains(addr), 10.146 - "address should be within the heap bounds"); 10.147 - 10.148 - if (!_nextMarkBitMap->isMarked(addr)) { 10.149 - _nextMarkBitMap->parMark(addr); 10.150 - } 10.151 -} 10.152 - 10.153 void ConcurrentMark::grayRegionIfNecessary(MemRegion mr) { 10.154 + guarantee(false, "grayRegionIfNecessary(): don't call this any more"); 10.155 + 10.156 // The objects on the region have already been marked "in bulk" by 10.157 // the caller. We only need to decide whether to push the region on 10.158 // the region stack or not. 10.159 @@ -1007,6 +1014,8 @@ 10.160 } 10.161 10.162 void ConcurrentMark::markAndGrayObjectIfNecessary(oop p) { 10.163 + guarantee(false, "markAndGrayObjectIfNecessary(): don't call this any more"); 10.164 + 10.165 // The object is not marked by the caller. We need to at least mark 10.166 // it and maybe push in on the stack. 10.167 10.168 @@ -1224,7 +1233,6 @@ 10.169 true /* expected_active */); 10.170 10.171 if (VerifyDuringGC) { 10.172 - 10.173 HandleMark hm; // handle scope 10.174 gclog_or_tty->print(" VerifyDuringGC:(after)"); 10.175 Universe::heap()->prepare_for_verify(); 10.176 @@ -1879,10 +1887,6 @@ 10.177 double end = os::elapsedTime(); 10.178 _cleanup_times.add((end - start) * 1000.0); 10.179 10.180 - // G1CollectedHeap::heap()->print(); 10.181 - // gclog_or_tty->print_cr("HEAP GC TIME STAMP : %d", 10.182 - // G1CollectedHeap::heap()->get_gc_time_stamp()); 10.183 - 10.184 if (PrintGC || PrintGCDetails) { 10.185 g1h->print_size_transition(gclog_or_tty, 10.186 start_used_bytes, 10.187 @@ -2669,6 +2673,8 @@ 10.188 } 10.189 10.190 void ConcurrentMark::drainAllSATBBuffers() { 10.191 + guarantee(false, "drainAllSATBBuffers(): don't call this any more"); 10.192 + 10.193 CMGlobalObjectClosure oc(this); 10.194 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 10.195 satb_mq_set.set_closure(&oc); 10.196 @@ -2687,12 +2693,6 @@ 10.197 assert(satb_mq_set.completed_buffers_num() == 0, "invariant"); 10.198 } 10.199 10.200 -void ConcurrentMark::markPrev(oop p) { 10.201 - // Note we are overriding the read-only view of the prev map here, via 10.202 - // the cast. 10.203 - ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*)p); 10.204 -} 10.205 - 10.206 void ConcurrentMark::clear(oop p) { 10.207 assert(p != NULL && p->is_oop(), "expected an oop"); 10.208 HeapWord* addr = (HeapWord*)p; 10.209 @@ -2702,13 +2702,21 @@ 10.210 _nextMarkBitMap->clear(addr); 10.211 } 10.212 10.213 -void ConcurrentMark::clearRangeBothMaps(MemRegion mr) { 10.214 +void ConcurrentMark::clearRangePrevBitmap(MemRegion mr) { 10.215 // Note we are overriding the read-only view of the prev map here, via 10.216 // the cast. 10.217 ((CMBitMap*)_prevMarkBitMap)->clearRange(mr); 10.218 +} 10.219 + 10.220 +void ConcurrentMark::clearRangeNextBitmap(MemRegion mr) { 10.221 _nextMarkBitMap->clearRange(mr); 10.222 } 10.223 10.224 +void ConcurrentMark::clearRangeBothBitmaps(MemRegion mr) { 10.225 + clearRangePrevBitmap(mr); 10.226 + clearRangeNextBitmap(mr); 10.227 +} 10.228 + 10.229 HeapRegion* 10.230 ConcurrentMark::claim_region(int task_num) { 10.231 // "checkpoint" the finger 10.232 @@ -2803,6 +2811,9 @@ 10.233 } 10.234 10.235 bool ConcurrentMark::invalidate_aborted_regions_in_cset() { 10.236 + guarantee(false, "invalidate_aborted_regions_in_cset(): " 10.237 + "don't call this any more"); 10.238 + 10.239 bool result = false; 10.240 for (int i = 0; i < (int)_max_task_num; ++i) { 10.241 CMTask* the_task = _tasks[i]; 10.242 @@ -2854,24 +2865,135 @@ 10.243 // ...then over the contents of the all the task queues. 10.244 queue->oops_do(cl); 10.245 } 10.246 - 10.247 - // Invalidate any entries, that are in the region stack, that 10.248 - // point into the collection set 10.249 - if (_regionStack.invalidate_entries_into_cset()) { 10.250 - // otherwise, any gray objects copied during the evacuation pause 10.251 - // might not be visited. 10.252 - assert(_should_gray_objects, "invariant"); 10.253 +} 10.254 + 10.255 +#ifndef PRODUCT 10.256 +enum VerifyNoCSetOopsPhase { 10.257 + VerifyNoCSetOopsStack, 10.258 + VerifyNoCSetOopsQueues, 10.259 + VerifyNoCSetOopsSATBCompleted, 10.260 + VerifyNoCSetOopsSATBThread 10.261 +}; 10.262 + 10.263 +class VerifyNoCSetOopsClosure : public OopClosure, public ObjectClosure { 10.264 +private: 10.265 + G1CollectedHeap* _g1h; 10.266 + VerifyNoCSetOopsPhase _phase; 10.267 + int _info; 10.268 + 10.269 + const char* phase_str() { 10.270 + switch (_phase) { 10.271 + case VerifyNoCSetOopsStack: return "Stack"; 10.272 + case VerifyNoCSetOopsQueues: return "Queue"; 10.273 + case VerifyNoCSetOopsSATBCompleted: return "Completed SATB Buffers"; 10.274 + case VerifyNoCSetOopsSATBThread: return "Thread SATB Buffers"; 10.275 + default: ShouldNotReachHere(); 10.276 + } 10.277 + return NULL; 10.278 } 10.279 10.280 - // Invalidate any aborted regions, recorded in the individual CM 10.281 - // tasks, that point into the collection set. 10.282 - if (invalidate_aborted_regions_in_cset()) { 10.283 - // otherwise, any gray objects copied during the evacuation pause 10.284 - // might not be visited. 10.285 - assert(_should_gray_objects, "invariant"); 10.286 + void do_object_work(oop obj) { 10.287 + guarantee(!_g1h->obj_in_cs(obj), 10.288 + err_msg("obj: "PTR_FORMAT" in CSet, phase: %s, info: %d", 10.289 + (void*) obj, phase_str(), _info)); 10.290 } 10.291 10.292 +public: 10.293 + VerifyNoCSetOopsClosure() : _g1h(G1CollectedHeap::heap()) { } 10.294 + 10.295 + void set_phase(VerifyNoCSetOopsPhase phase, int info = -1) { 10.296 + _phase = phase; 10.297 + _info = info; 10.298 + } 10.299 + 10.300 + virtual void do_oop(oop* p) { 10.301 + oop obj = oopDesc::load_decode_heap_oop(p); 10.302 + do_object_work(obj); 10.303 + } 10.304 + 10.305 + virtual void do_oop(narrowOop* p) { 10.306 + // We should not come across narrow oops while scanning marking 10.307 + // stacks and SATB buffers. 10.308 + ShouldNotReachHere(); 10.309 + } 10.310 + 10.311 + virtual void do_object(oop obj) { 10.312 + do_object_work(obj); 10.313 + } 10.314 +}; 10.315 + 10.316 +void ConcurrentMark::verify_no_cset_oops(bool verify_stacks, 10.317 + bool verify_enqueued_buffers, 10.318 + bool verify_thread_buffers, 10.319 + bool verify_fingers) { 10.320 + assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint"); 10.321 + if (!G1CollectedHeap::heap()->mark_in_progress()) { 10.322 + return; 10.323 + } 10.324 + 10.325 + VerifyNoCSetOopsClosure cl; 10.326 + 10.327 + if (verify_stacks) { 10.328 + // Verify entries on the global mark stack 10.329 + cl.set_phase(VerifyNoCSetOopsStack); 10.330 + _markStack.oops_do(&cl); 10.331 + 10.332 + // Verify entries on the task queues 10.333 + for (int i = 0; i < (int) _max_task_num; i += 1) { 10.334 + cl.set_phase(VerifyNoCSetOopsQueues, i); 10.335 + OopTaskQueue* queue = _task_queues->queue(i); 10.336 + queue->oops_do(&cl); 10.337 + } 10.338 + } 10.339 + 10.340 + SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); 10.341 + 10.342 + // Verify entries on the enqueued SATB buffers 10.343 + if (verify_enqueued_buffers) { 10.344 + cl.set_phase(VerifyNoCSetOopsSATBCompleted); 10.345 + satb_qs.iterate_completed_buffers_read_only(&cl); 10.346 + } 10.347 + 10.348 + // Verify entries on the per-thread SATB buffers 10.349 + if (verify_thread_buffers) { 10.350 + cl.set_phase(VerifyNoCSetOopsSATBThread); 10.351 + satb_qs.iterate_thread_buffers_read_only(&cl); 10.352 + } 10.353 + 10.354 + if (verify_fingers) { 10.355 + // Verify the global finger 10.356 + HeapWord* global_finger = finger(); 10.357 + if (global_finger != NULL && global_finger < _heap_end) { 10.358 + // The global finger always points to a heap region boundary. We 10.359 + // use heap_region_containing_raw() to get the containing region 10.360 + // given that the global finger could be pointing to a free region 10.361 + // which subsequently becomes continues humongous. If that 10.362 + // happens, heap_region_containing() will return the bottom of the 10.363 + // corresponding starts humongous region and the check below will 10.364 + // not hold any more. 10.365 + HeapRegion* global_hr = _g1h->heap_region_containing_raw(global_finger); 10.366 + guarantee(global_finger == global_hr->bottom(), 10.367 + err_msg("global finger: "PTR_FORMAT" region: "HR_FORMAT, 10.368 + global_finger, HR_FORMAT_PARAMS(global_hr))); 10.369 + } 10.370 + 10.371 + // Verify the task fingers 10.372 + assert(parallel_marking_threads() <= _max_task_num, "sanity"); 10.373 + for (int i = 0; i < (int) parallel_marking_threads(); i += 1) { 10.374 + CMTask* task = _tasks[i]; 10.375 + HeapWord* task_finger = task->finger(); 10.376 + if (task_finger != NULL && task_finger < _heap_end) { 10.377 + // See above note on the global finger verification. 10.378 + HeapRegion* task_hr = _g1h->heap_region_containing_raw(task_finger); 10.379 + guarantee(task_finger == task_hr->bottom() || 10.380 + !task_hr->in_collection_set(), 10.381 + err_msg("task finger: "PTR_FORMAT" region: "HR_FORMAT, 10.382 + task_finger, HR_FORMAT_PARAMS(task_hr))); 10.383 + } 10.384 + } 10.385 + } 10.386 } 10.387 +#endif // PRODUCT 10.388 10.389 void ConcurrentMark::clear_marking_state(bool clear_overflow) { 10.390 _markStack.setEmpty(); 10.391 @@ -3080,19 +3202,6 @@ 10.392 } 10.393 }; 10.394 10.395 -class SetClaimValuesInCSetHRClosure: public HeapRegionClosure { 10.396 - jint _claim_value; 10.397 - 10.398 -public: 10.399 - SetClaimValuesInCSetHRClosure(jint claim_value) : 10.400 - _claim_value(claim_value) { } 10.401 - 10.402 - bool doHeapRegion(HeapRegion* hr) { 10.403 - hr->set_claim_value(_claim_value); 10.404 - return false; 10.405 - } 10.406 -}; 10.407 - 10.408 class G1ParCompleteMarkInCSetTask: public AbstractGangTask { 10.409 protected: 10.410 G1CollectedHeap* _g1h; 10.411 @@ -3112,6 +3221,9 @@ 10.412 }; 10.413 10.414 void ConcurrentMark::complete_marking_in_collection_set() { 10.415 + guarantee(false, "complete_marking_in_collection_set(): " 10.416 + "don't call this any more"); 10.417 + 10.418 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 10.419 10.420 if (!g1h->mark_in_progress()) { 10.421 @@ -3135,9 +3247,8 @@ 10.422 10.423 assert(g1h->check_cset_heap_region_claim_values(HeapRegion::CompleteMarkCSetClaimValue), "sanity"); 10.424 10.425 - // Now reset the claim values in the regions in the collection set. 10.426 - SetClaimValuesInCSetHRClosure set_cv_cl(HeapRegion::InitialClaimValue); 10.427 - g1h->collection_set_iterate(&set_cv_cl); 10.428 + // Reset the claim values in the regions in the collection set. 10.429 + g1h->reset_cset_heap_region_claim_values(); 10.430 10.431 assert(g1h->check_cset_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity"); 10.432 10.433 @@ -3160,6 +3271,8 @@ 10.434 // newCSet(). 10.435 10.436 void ConcurrentMark::newCSet() { 10.437 + guarantee(false, "newCSet(): don't call this any more"); 10.438 + 10.439 if (!concurrent_marking_in_progress()) { 10.440 // nothing to do if marking is not in progress 10.441 return; 10.442 @@ -3198,6 +3311,8 @@ 10.443 } 10.444 10.445 void ConcurrentMark::registerCSetRegion(HeapRegion* hr) { 10.446 + guarantee(false, "registerCSetRegion(): don't call this any more"); 10.447 + 10.448 if (!concurrent_marking_in_progress()) return; 10.449 10.450 HeapWord* region_end = hr->end(); 10.451 @@ -3209,6 +3324,9 @@ 10.452 // Resets the region fields of active CMTasks whose values point 10.453 // into the collection set. 10.454 void ConcurrentMark::reset_active_task_region_fields_in_cset() { 10.455 + guarantee(false, "reset_active_task_region_fields_in_cset(): " 10.456 + "don't call this any more"); 10.457 + 10.458 assert(SafepointSynchronize::is_at_safepoint(), "should be in STW"); 10.459 assert(parallel_marking_threads() <= _max_task_num, "sanity"); 10.460 10.461 @@ -3919,6 +4037,10 @@ 10.462 } 10.463 10.464 void CMTask::drain_region_stack(BitMapClosure* bc) { 10.465 + assert(_cm->region_stack_empty(), "region stack should be empty"); 10.466 + assert(_aborted_region.is_empty(), "aborted region should be empty"); 10.467 + return; 10.468 + 10.469 if (has_aborted()) return; 10.470 10.471 assert(_region_finger == NULL,
11.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp Tue Jan 17 13:08:52 2012 -0500 11.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp Tue Jan 17 21:25:28 2012 -0500 11.3 @@ -1,5 +1,5 @@ 11.4 /* 11.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 11.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 11.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 11.8 * 11.9 * This code is free software; you can redistribute it and/or modify it 11.10 @@ -166,10 +166,10 @@ 11.11 // Ideally this should be GrowableArray<> just like MSC's marking stack(s). 11.12 class CMMarkStack VALUE_OBJ_CLASS_SPEC { 11.13 ConcurrentMark* _cm; 11.14 - oop* _base; // bottom of stack 11.15 - jint _index; // one more than last occupied index 11.16 - jint _capacity; // max #elements 11.17 - jint _oops_do_bound; // Number of elements to include in next iteration. 11.18 + oop* _base; // bottom of stack 11.19 + jint _index; // one more than last occupied index 11.20 + jint _capacity; // max #elements 11.21 + jint _saved_index; // value of _index saved at start of GC 11.22 NOT_PRODUCT(jint _max_depth;) // max depth plumbed during run 11.23 11.24 bool _overflow; 11.25 @@ -247,16 +247,12 @@ 11.26 11.27 void setEmpty() { _index = 0; clear_overflow(); } 11.28 11.29 - // Record the current size; a subsequent "oops_do" will iterate only over 11.30 - // indices valid at the time of this call. 11.31 - void set_oops_do_bound(jint bound = -1) { 11.32 - if (bound == -1) { 11.33 - _oops_do_bound = _index; 11.34 - } else { 11.35 - _oops_do_bound = bound; 11.36 - } 11.37 - } 11.38 - jint oops_do_bound() { return _oops_do_bound; } 11.39 + // Record the current index. 11.40 + void note_start_of_gc(); 11.41 + 11.42 + // Make sure that we have not added any entries to the stack during GC. 11.43 + void note_end_of_gc(); 11.44 + 11.45 // iterate over the oops in the mark stack, up to the bound recorded via 11.46 // the call above. 11.47 void oops_do(OopClosure* f); 11.48 @@ -724,10 +720,9 @@ 11.49 // G1CollectedHeap 11.50 11.51 // This notifies CM that a root during initial-mark needs to be 11.52 - // grayed and it's MT-safe. Currently, we just mark it. But, in the 11.53 - // future, we can experiment with pushing it on the stack and we can 11.54 - // do this without changing G1CollectedHeap. 11.55 - void grayRoot(oop p); 11.56 + // grayed. It is MT-safe. 11.57 + inline void grayRoot(oop obj, size_t word_size); 11.58 + 11.59 // It's used during evacuation pauses to gray a region, if 11.60 // necessary, and it's MT-safe. It assumes that the caller has 11.61 // marked any objects on that region. If _should_gray_objects is 11.62 @@ -735,6 +730,7 @@ 11.63 // pushed on the region stack, if it is located below the global 11.64 // finger, otherwise we do nothing. 11.65 void grayRegionIfNecessary(MemRegion mr); 11.66 + 11.67 // It's used during evacuation pauses to mark and, if necessary, 11.68 // gray a single object and it's MT-safe. It assumes the caller did 11.69 // not mark the object. If _should_gray_objects is true and we're 11.70 @@ -791,24 +787,40 @@ 11.71 11.72 // Mark in the previous bitmap. NB: this is usually read-only, so use 11.73 // this carefully! 11.74 - void markPrev(oop p); 11.75 + inline void markPrev(oop p); 11.76 + inline void markNext(oop p); 11.77 void clear(oop p); 11.78 - // Clears marks for all objects in the given range, for both prev and 11.79 - // next bitmaps. NB: the previous bitmap is usually read-only, so use 11.80 - // this carefully! 11.81 - void clearRangeBothMaps(MemRegion mr); 11.82 + // Clears marks for all objects in the given range, for the prev, 11.83 + // next, or both bitmaps. NB: the previous bitmap is usually 11.84 + // read-only, so use this carefully! 11.85 + void clearRangePrevBitmap(MemRegion mr); 11.86 + void clearRangeNextBitmap(MemRegion mr); 11.87 + void clearRangeBothBitmaps(MemRegion mr); 11.88 11.89 - // Record the current top of the mark and region stacks; a 11.90 - // subsequent oops_do() on the mark stack and 11.91 - // invalidate_entries_into_cset() on the region stack will iterate 11.92 - // only over indices valid at the time of this call. 11.93 - void set_oops_do_bound() { 11.94 - _markStack.set_oops_do_bound(); 11.95 - _regionStack.set_oops_do_bound(); 11.96 + // Notify data structures that a GC has started. 11.97 + void note_start_of_gc() { 11.98 + _markStack.note_start_of_gc(); 11.99 } 11.100 + 11.101 + // Notify data structures that a GC is finished. 11.102 + void note_end_of_gc() { 11.103 + _markStack.note_end_of_gc(); 11.104 + } 11.105 + 11.106 // Iterate over the oops in the mark stack and all local queues. It 11.107 // also calls invalidate_entries_into_cset() on the region stack. 11.108 void oops_do(OopClosure* f); 11.109 + 11.110 + // Verify that there are no CSet oops on the stacks (taskqueues / 11.111 + // global mark stack), enqueued SATB buffers, per-thread SATB 11.112 + // buffers, and fingers (global / per-task). The boolean parameters 11.113 + // decide which of the above data structures to verify. If marking 11.114 + // is not in progress, it's a no-op. 11.115 + void verify_no_cset_oops(bool verify_stacks, 11.116 + bool verify_enqueued_buffers, 11.117 + bool verify_thread_buffers, 11.118 + bool verify_fingers) PRODUCT_RETURN; 11.119 + 11.120 // It is called at the end of an evacuation pause during marking so 11.121 // that CM is notified of where the new end of the heap is. It 11.122 // doesn't do anything if concurrent_marking_in_progress() is false, 11.123 @@ -1166,6 +1178,7 @@ 11.124 // It keeps picking SATB buffers and processing them until no SATB 11.125 // buffers are available. 11.126 void drain_satb_buffers(); 11.127 + 11.128 // It keeps popping regions from the region stack and processing 11.129 // them until the region stack is empty. 11.130 void drain_region_stack(BitMapClosure* closure);
12.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp Tue Jan 17 13:08:52 2012 -0500 12.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp Tue Jan 17 21:25:28 2012 -0500 12.3 @@ -1,5 +1,5 @@ 12.4 /* 12.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 12.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 12.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 12.8 * 12.9 * This code is free software; you can redistribute it and/or modify it 12.10 @@ -153,4 +153,46 @@ 12.11 } 12.12 } 12.13 12.14 +inline void ConcurrentMark::markPrev(oop p) { 12.15 + assert(!_prevMarkBitMap->isMarked((HeapWord*) p), "sanity"); 12.16 + // Note we are overriding the read-only view of the prev map here, via 12.17 + // the cast. 12.18 + ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*) p); 12.19 +} 12.20 + 12.21 +inline void ConcurrentMark::markNext(oop p) { 12.22 + assert(!_nextMarkBitMap->isMarked((HeapWord*) p), "sanity"); 12.23 + _nextMarkBitMap->mark((HeapWord*) p); 12.24 +} 12.25 + 12.26 +inline void ConcurrentMark::grayRoot(oop obj, size_t word_size) { 12.27 + HeapWord* addr = (HeapWord*) obj; 12.28 + 12.29 + // Currently we don't do anything with word_size but we will use it 12.30 + // in the very near future in the liveness calculation piggy-backing 12.31 + // changes. 12.32 + 12.33 +#ifdef ASSERT 12.34 + HeapRegion* hr = _g1h->heap_region_containing(addr); 12.35 + assert(hr != NULL, "sanity"); 12.36 + assert(!hr->is_survivor(), "should not allocate survivors during IM"); 12.37 + assert(addr < hr->next_top_at_mark_start(), 12.38 + err_msg("addr: "PTR_FORMAT" hr: "HR_FORMAT" NTAMS: "PTR_FORMAT, 12.39 + addr, HR_FORMAT_PARAMS(hr), hr->next_top_at_mark_start())); 12.40 + // We cannot assert that word_size == obj->size() given that obj 12.41 + // might not be in a consistent state (another thread might be in 12.42 + // the process of copying it). So the best thing we can do is to 12.43 + // assert that word_size is under an upper bound which is its 12.44 + // containing region's capacity. 12.45 + assert(word_size * HeapWordSize <= hr->capacity(), 12.46 + err_msg("size: "SIZE_FORMAT" capacity: "SIZE_FORMAT" "HR_FORMAT, 12.47 + word_size * HeapWordSize, hr->capacity(), 12.48 + HR_FORMAT_PARAMS(hr))); 12.49 +#endif // ASSERT 12.50 + 12.51 + if (!_nextMarkBitMap->isMarked(addr)) { 12.52 + _nextMarkBitMap->parMark(addr); 12.53 + } 12.54 +} 12.55 + 12.56 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP
13.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Tue Jan 17 13:08:52 2012 -0500 13.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Tue Jan 17 21:25:28 2012 -0500 13.3 @@ -1,5 +1,5 @@ 13.4 /* 13.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 13.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 13.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 13.8 * 13.9 * This code is free software; you can redistribute it and/or modify it 13.10 @@ -32,9 +32,11 @@ 13.11 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" 13.12 #include "gc_implementation/g1/g1CollectorPolicy.hpp" 13.13 #include "gc_implementation/g1/g1ErgoVerbose.hpp" 13.14 +#include "gc_implementation/g1/g1EvacFailure.hpp" 13.15 #include "gc_implementation/g1/g1MarkSweep.hpp" 13.16 #include "gc_implementation/g1/g1OopClosures.inline.hpp" 13.17 #include "gc_implementation/g1/g1RemSet.inline.hpp" 13.18 +#include "gc_implementation/g1/heapRegion.inline.hpp" 13.19 #include "gc_implementation/g1/heapRegionRemSet.hpp" 13.20 #include "gc_implementation/g1/heapRegionSeq.inline.hpp" 13.21 #include "gc_implementation/g1/vm_operations_g1.hpp" 13.22 @@ -591,17 +593,29 @@ 13.23 } 13.24 res = new_region_try_secondary_free_list(); 13.25 } 13.26 - if (res == NULL && do_expand) { 13.27 + if (res == NULL && do_expand && _expand_heap_after_alloc_failure) { 13.28 + // Currently, only attempts to allocate GC alloc regions set 13.29 + // do_expand to true. So, we should only reach here during a 13.30 + // safepoint. If this assumption changes we might have to 13.31 + // reconsider the use of _expand_heap_after_alloc_failure. 13.32 + assert(SafepointSynchronize::is_at_safepoint(), "invariant"); 13.33 + 13.34 ergo_verbose1(ErgoHeapSizing, 13.35 "attempt heap expansion", 13.36 ergo_format_reason("region allocation request failed") 13.37 ergo_format_byte("allocation request"), 13.38 word_size * HeapWordSize); 13.39 if (expand(word_size * HeapWordSize)) { 13.40 - // Even though the heap was expanded, it might not have reached 13.41 - // the desired size. So, we cannot assume that the allocation 13.42 - // will succeed. 13.43 + // Given that expand() succeeded in expanding the heap, and we 13.44 + // always expand the heap by an amount aligned to the heap 13.45 + // region size, the free list should in theory not be empty. So 13.46 + // it would probably be OK to use remove_head(). But the extra 13.47 + // check for NULL is unlikely to be a performance issue here (we 13.48 + // just expanded the heap!) so let's just be conservative and 13.49 + // use remove_head_or_null(). 13.50 res = _free_list.remove_head_or_null(); 13.51 + } else { 13.52 + _expand_heap_after_alloc_failure = false; 13.53 } 13.54 } 13.55 return res; 13.56 @@ -1838,6 +1852,7 @@ 13.57 _young_list(new YoungList(this)), 13.58 _gc_time_stamp(0), 13.59 _retained_old_gc_alloc_region(NULL), 13.60 + _expand_heap_after_alloc_failure(true), 13.61 _surviving_young_words(NULL), 13.62 _full_collections_completed(0), 13.63 _in_cset_fast_test(NULL), 13.64 @@ -2605,12 +2620,16 @@ 13.65 } 13.66 }; 13.67 13.68 -void 13.69 -G1CollectedHeap::reset_heap_region_claim_values() { 13.70 +void G1CollectedHeap::reset_heap_region_claim_values() { 13.71 ResetClaimValuesClosure blk; 13.72 heap_region_iterate(&blk); 13.73 } 13.74 13.75 +void G1CollectedHeap::reset_cset_heap_region_claim_values() { 13.76 + ResetClaimValuesClosure blk; 13.77 + collection_set_iterate(&blk); 13.78 +} 13.79 + 13.80 #ifdef ASSERT 13.81 // This checks whether all regions in the heap have the correct claim 13.82 // value. I also piggy-backed on this a check to ensure that the 13.83 @@ -3000,14 +3019,20 @@ 13.84 } else { 13.85 VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo); 13.86 r->object_iterate(¬_dead_yet_cl); 13.87 - if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) { 13.88 - gclog_or_tty->print_cr("["PTR_FORMAT","PTR_FORMAT"] " 13.89 - "max_live_bytes "SIZE_FORMAT" " 13.90 - "< calculated "SIZE_FORMAT, 13.91 - r->bottom(), r->end(), 13.92 - r->max_live_bytes(), 13.93 + if (_vo != VerifyOption_G1UseNextMarking) { 13.94 + if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) { 13.95 + gclog_or_tty->print_cr("["PTR_FORMAT","PTR_FORMAT"] " 13.96 + "max_live_bytes "SIZE_FORMAT" " 13.97 + "< calculated "SIZE_FORMAT, 13.98 + r->bottom(), r->end(), 13.99 + r->max_live_bytes(), 13.100 not_dead_yet_cl.live_bytes()); 13.101 - _failures = true; 13.102 + _failures = true; 13.103 + } 13.104 + } else { 13.105 + // When vo == UseNextMarking we cannot currently do a sanity 13.106 + // check on the live bytes as the calculation has not been 13.107 + // finalized yet. 13.108 } 13.109 } 13.110 } 13.111 @@ -3641,25 +3666,6 @@ 13.112 } 13.113 perm_gen()->save_marks(); 13.114 13.115 - // We must do this before any possible evacuation that should propagate 13.116 - // marks. 13.117 - if (mark_in_progress()) { 13.118 - double start_time_sec = os::elapsedTime(); 13.119 - 13.120 - _cm->drainAllSATBBuffers(); 13.121 - double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0; 13.122 - g1_policy()->record_satb_drain_time(finish_mark_ms); 13.123 - } 13.124 - // Record the number of elements currently on the mark stack, so we 13.125 - // only iterate over these. (Since evacuation may add to the mark 13.126 - // stack, doing more exposes race conditions.) If no mark is in 13.127 - // progress, this will be zero. 13.128 - _cm->set_oops_do_bound(); 13.129 - 13.130 - if (mark_in_progress()) { 13.131 - concurrent_mark()->newCSet(); 13.132 - } 13.133 - 13.134 #if YOUNG_LIST_VERBOSE 13.135 gclog_or_tty->print_cr("\nBefore choosing collection set.\nYoung_list:"); 13.136 _young_list->print(); 13.137 @@ -3668,6 +3674,16 @@ 13.138 13.139 g1_policy()->choose_collection_set(target_pause_time_ms); 13.140 13.141 + _cm->note_start_of_gc(); 13.142 + // We should not verify the per-thread SATB buffers given that 13.143 + // we have not filtered them yet (we'll do so during the 13.144 + // GC). We also call this after choose_collection_set() to 13.145 + // ensure that the CSet has been finalized. 13.146 + _cm->verify_no_cset_oops(true /* verify_stacks */, 13.147 + true /* verify_enqueued_buffers */, 13.148 + false /* verify_thread_buffers */, 13.149 + true /* verify_fingers */); 13.150 + 13.151 if (_hr_printer.is_active()) { 13.152 HeapRegion* hr = g1_policy()->collection_set(); 13.153 while (hr != NULL) { 13.154 @@ -3684,16 +3700,6 @@ 13.155 } 13.156 } 13.157 13.158 - // We have chosen the complete collection set. If marking is 13.159 - // active then, we clear the region fields of any of the 13.160 - // concurrent marking tasks whose region fields point into 13.161 - // the collection set as these values will become stale. This 13.162 - // will cause the owning marking threads to claim a new region 13.163 - // when marking restarts. 13.164 - if (mark_in_progress()) { 13.165 - concurrent_mark()->reset_active_task_region_fields_in_cset(); 13.166 - } 13.167 - 13.168 #ifdef ASSERT 13.169 VerifyCSetClosure cl; 13.170 collection_set_iterate(&cl); 13.171 @@ -3707,6 +3713,16 @@ 13.172 // Actually do the work... 13.173 evacuate_collection_set(); 13.174 13.175 + // We do this to mainly verify the per-thread SATB buffers 13.176 + // (which have been filtered by now) since we didn't verify 13.177 + // them earlier. No point in re-checking the stacks / enqueued 13.178 + // buffers given that the CSet has not changed since last time 13.179 + // we checked. 13.180 + _cm->verify_no_cset_oops(false /* verify_stacks */, 13.181 + false /* verify_enqueued_buffers */, 13.182 + true /* verify_thread_buffers */, 13.183 + true /* verify_fingers */); 13.184 + 13.185 free_collection_set(g1_policy()->collection_set()); 13.186 g1_policy()->clear_collection_set(); 13.187 13.188 @@ -3775,6 +3791,8 @@ 13.189 size_t expand_bytes = g1_policy()->expansion_amount(); 13.190 if (expand_bytes > 0) { 13.191 size_t bytes_before = capacity(); 13.192 + // No need for an ergo verbose message here, 13.193 + // expansion_amount() does this when it returns a value > 0. 13.194 if (!expand(expand_bytes)) { 13.195 // We failed to expand the heap so let's verify that 13.196 // committed/uncommitted amount match the backing store 13.197 @@ -3784,6 +3802,14 @@ 13.198 } 13.199 } 13.200 13.201 + // We redo the verificaiton but now wrt to the new CSet which 13.202 + // has just got initialized after the previous CSet was freed. 13.203 + _cm->verify_no_cset_oops(true /* verify_stacks */, 13.204 + true /* verify_enqueued_buffers */, 13.205 + true /* verify_thread_buffers */, 13.206 + true /* verify_fingers */); 13.207 + _cm->note_end_of_gc(); 13.208 + 13.209 double end_time_sec = os::elapsedTime(); 13.210 double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS; 13.211 g1_policy()->record_pause_time_ms(pause_time_ms); 13.212 @@ -3831,21 +3857,6 @@ 13.213 // CM reference discovery will be re-enabled if necessary. 13.214 } 13.215 13.216 - { 13.217 - size_t expand_bytes = g1_policy()->expansion_amount(); 13.218 - if (expand_bytes > 0) { 13.219 - size_t bytes_before = capacity(); 13.220 - // No need for an ergo verbose message here, 13.221 - // expansion_amount() does this when it returns a value > 0. 13.222 - if (!expand(expand_bytes)) { 13.223 - // We failed to expand the heap so let's verify that 13.224 - // committed/uncommitted amount match the backing store 13.225 - assert(capacity() == _g1_storage.committed_size(), "committed size mismatch"); 13.226 - assert(max_capacity() == _g1_storage.reserved_size(), "reserved size mismatch"); 13.227 - } 13.228 - } 13.229 - } 13.230 - 13.231 // We should do this after we potentially expand the heap so 13.232 // that all the COMMIT events are generated before the end GC 13.233 // event, and after we retire the GC alloc regions so that all 13.234 @@ -3949,6 +3960,8 @@ 13.235 // we allocate to in the region sets. We'll re-add it later, when 13.236 // it's retired again. 13.237 _old_set.remove(retained_region); 13.238 + bool during_im = g1_policy()->during_initial_mark_pause(); 13.239 + retained_region->note_start_of_copying(during_im); 13.240 _old_gc_alloc_region.set(retained_region); 13.241 _hr_printer.reuse(retained_region); 13.242 } 13.243 @@ -3985,157 +3998,26 @@ 13.244 _evac_failure_scan_stack = NULL; 13.245 } 13.246 13.247 -class UpdateRSetDeferred : public OopsInHeapRegionClosure { 13.248 -private: 13.249 - G1CollectedHeap* _g1; 13.250 - DirtyCardQueue *_dcq; 13.251 - CardTableModRefBS* _ct_bs; 13.252 - 13.253 -public: 13.254 - UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) : 13.255 - _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {} 13.256 - 13.257 - virtual void do_oop(narrowOop* p) { do_oop_work(p); } 13.258 - virtual void do_oop( oop* p) { do_oop_work(p); } 13.259 - template <class T> void do_oop_work(T* p) { 13.260 - assert(_from->is_in_reserved(p), "paranoia"); 13.261 - if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && 13.262 - !_from->is_survivor()) { 13.263 - size_t card_index = _ct_bs->index_for(p); 13.264 - if (_ct_bs->mark_card_deferred(card_index)) { 13.265 - _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index)); 13.266 - } 13.267 - } 13.268 - } 13.269 -}; 13.270 - 13.271 -class RemoveSelfPointerClosure: public ObjectClosure { 13.272 -private: 13.273 - G1CollectedHeap* _g1; 13.274 - ConcurrentMark* _cm; 13.275 - HeapRegion* _hr; 13.276 - size_t _prev_marked_bytes; 13.277 - size_t _next_marked_bytes; 13.278 - OopsInHeapRegionClosure *_cl; 13.279 -public: 13.280 - RemoveSelfPointerClosure(G1CollectedHeap* g1, HeapRegion* hr, 13.281 - OopsInHeapRegionClosure* cl) : 13.282 - _g1(g1), _hr(hr), _cm(_g1->concurrent_mark()), _prev_marked_bytes(0), 13.283 - _next_marked_bytes(0), _cl(cl) {} 13.284 - 13.285 - size_t prev_marked_bytes() { return _prev_marked_bytes; } 13.286 - size_t next_marked_bytes() { return _next_marked_bytes; } 13.287 - 13.288 - // <original comment> 13.289 - // The original idea here was to coalesce evacuated and dead objects. 13.290 - // However that caused complications with the block offset table (BOT). 13.291 - // In particular if there were two TLABs, one of them partially refined. 13.292 - // |----- TLAB_1--------|----TLAB_2-~~~(partially refined part)~~~| 13.293 - // The BOT entries of the unrefined part of TLAB_2 point to the start 13.294 - // of TLAB_2. If the last object of the TLAB_1 and the first object 13.295 - // of TLAB_2 are coalesced, then the cards of the unrefined part 13.296 - // would point into middle of the filler object. 13.297 - // The current approach is to not coalesce and leave the BOT contents intact. 13.298 - // </original comment> 13.299 - // 13.300 - // We now reset the BOT when we start the object iteration over the 13.301 - // region and refine its entries for every object we come across. So 13.302 - // the above comment is not really relevant and we should be able 13.303 - // to coalesce dead objects if we want to. 13.304 - void do_object(oop obj) { 13.305 - HeapWord* obj_addr = (HeapWord*) obj; 13.306 - assert(_hr->is_in(obj_addr), "sanity"); 13.307 - size_t obj_size = obj->size(); 13.308 - _hr->update_bot_for_object(obj_addr, obj_size); 13.309 - if (obj->is_forwarded() && obj->forwardee() == obj) { 13.310 - // The object failed to move. 13.311 - assert(!_g1->is_obj_dead(obj), "We should not be preserving dead objs."); 13.312 - _cm->markPrev(obj); 13.313 - assert(_cm->isPrevMarked(obj), "Should be marked!"); 13.314 - _prev_marked_bytes += (obj_size * HeapWordSize); 13.315 - if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) { 13.316 - _cm->markAndGrayObjectIfNecessary(obj); 13.317 - } 13.318 - obj->set_mark(markOopDesc::prototype()); 13.319 - // While we were processing RSet buffers during the 13.320 - // collection, we actually didn't scan any cards on the 13.321 - // collection set, since we didn't want to update remebered 13.322 - // sets with entries that point into the collection set, given 13.323 - // that live objects fromthe collection set are about to move 13.324 - // and such entries will be stale very soon. This change also 13.325 - // dealt with a reliability issue which involved scanning a 13.326 - // card in the collection set and coming across an array that 13.327 - // was being chunked and looking malformed. The problem is 13.328 - // that, if evacuation fails, we might have remembered set 13.329 - // entries missing given that we skipped cards on the 13.330 - // collection set. So, we'll recreate such entries now. 13.331 - obj->oop_iterate(_cl); 13.332 - assert(_cm->isPrevMarked(obj), "Should be marked!"); 13.333 - } else { 13.334 - // The object has been either evacuated or is dead. Fill it with a 13.335 - // dummy object. 13.336 - MemRegion mr((HeapWord*)obj, obj_size); 13.337 - CollectedHeap::fill_with_object(mr); 13.338 - _cm->clearRangeBothMaps(mr); 13.339 - } 13.340 - } 13.341 -}; 13.342 - 13.343 void G1CollectedHeap::remove_self_forwarding_pointers() { 13.344 - UpdateRSetImmediate immediate_update(_g1h->g1_rem_set()); 13.345 - DirtyCardQueue dcq(&_g1h->dirty_card_queue_set()); 13.346 - UpdateRSetDeferred deferred_update(_g1h, &dcq); 13.347 - OopsInHeapRegionClosure *cl; 13.348 - if (G1DeferredRSUpdate) { 13.349 - cl = &deferred_update; 13.350 + assert(check_cset_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity"); 13.351 + assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); 13.352 + 13.353 + G1ParRemoveSelfForwardPtrsTask rsfp_task(this); 13.354 + 13.355 + if (G1CollectedHeap::use_parallel_gc_threads()) { 13.356 + set_par_threads(); 13.357 + workers()->run_task(&rsfp_task); 13.358 + set_par_threads(0); 13.359 } else { 13.360 - cl = &immediate_update; 13.361 - } 13.362 - HeapRegion* cur = g1_policy()->collection_set(); 13.363 - while (cur != NULL) { 13.364 - assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); 13.365 - assert(!cur->isHumongous(), "sanity"); 13.366 - 13.367 - if (cur->evacuation_failed()) { 13.368 - assert(cur->in_collection_set(), "bad CS"); 13.369 - RemoveSelfPointerClosure rspc(_g1h, cur, cl); 13.370 - 13.371 - // In the common case we make sure that this is done when the 13.372 - // region is freed so that it is "ready-to-go" when it's 13.373 - // re-allocated. However, when evacuation failure happens, a 13.374 - // region will remain in the heap and might ultimately be added 13.375 - // to a CSet in the future. So we have to be careful here and 13.376 - // make sure the region's RSet is ready for parallel iteration 13.377 - // whenever this might be required in the future. 13.378 - cur->rem_set()->reset_for_par_iteration(); 13.379 - cur->reset_bot(); 13.380 - cl->set_region(cur); 13.381 - cur->object_iterate(&rspc); 13.382 - 13.383 - // A number of manipulations to make the TAMS be the current top, 13.384 - // and the marked bytes be the ones observed in the iteration. 13.385 - if (_g1h->concurrent_mark()->at_least_one_mark_complete()) { 13.386 - // The comments below are the postconditions achieved by the 13.387 - // calls. Note especially the last such condition, which says that 13.388 - // the count of marked bytes has been properly restored. 13.389 - cur->note_start_of_marking(false); 13.390 - // _next_top_at_mark_start == top, _next_marked_bytes == 0 13.391 - cur->add_to_marked_bytes(rspc.prev_marked_bytes()); 13.392 - // _next_marked_bytes == prev_marked_bytes. 13.393 - cur->note_end_of_marking(); 13.394 - // _prev_top_at_mark_start == top(), 13.395 - // _prev_marked_bytes == prev_marked_bytes 13.396 - } 13.397 - // If there is no mark in progress, we modified the _next variables 13.398 - // above needlessly, but harmlessly. 13.399 - if (_g1h->mark_in_progress()) { 13.400 - cur->note_start_of_marking(false); 13.401 - // _next_top_at_mark_start == top, _next_marked_bytes == 0 13.402 - // _next_marked_bytes == next_marked_bytes. 13.403 - } 13.404 - } 13.405 - cur = cur->next_in_collection_set(); 13.406 - } 13.407 + rsfp_task.work(0); 13.408 + } 13.409 + 13.410 + assert(check_cset_heap_region_claim_values(HeapRegion::ParEvacFailureClaimValue), "sanity"); 13.411 + 13.412 + // Reset the claim values in the regions in the collection set. 13.413 + reset_cset_heap_region_claim_values(); 13.414 + 13.415 + assert(check_cset_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity"); 13.416 assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); 13.417 13.418 // Now restore saved marks, if any. 13.419 @@ -4148,6 +4030,7 @@ 13.420 markOop m = _preserved_marks_of_objs->at(i); 13.421 obj->set_mark(m); 13.422 } 13.423 + 13.424 // Delete the preserved marks growable arrays (allocated on the C heap). 13.425 delete _objs_with_preserved_marks; 13.426 delete _preserved_marks_of_objs; 13.427 @@ -4172,8 +4055,7 @@ 13.428 13.429 oop 13.430 G1CollectedHeap::handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, 13.431 - oop old, 13.432 - bool should_mark_root) { 13.433 + oop old) { 13.434 assert(obj_in_cs(old), 13.435 err_msg("obj: "PTR_FORMAT" should still be in the CSet", 13.436 (HeapWord*) old)); 13.437 @@ -4182,15 +4064,6 @@ 13.438 if (forward_ptr == NULL) { 13.439 // Forward-to-self succeeded. 13.440 13.441 - // should_mark_root will be true when this routine is called 13.442 - // from a root scanning closure during an initial mark pause. 13.443 - // In this case the thread that succeeds in self-forwarding the 13.444 - // object is also responsible for marking the object. 13.445 - if (should_mark_root) { 13.446 - assert(!oopDesc::is_null(old), "shouldn't be"); 13.447 - _cm->grayRoot(old); 13.448 - } 13.449 - 13.450 if (_evac_failure_closure != cl) { 13.451 MutexLockerEx x(EvacFailureStack_lock, Mutex::_no_safepoint_check_flag); 13.452 assert(!_drain_in_progress, 13.453 @@ -4286,30 +4159,8 @@ 13.454 return NULL; 13.455 } 13.456 13.457 -#ifndef PRODUCT 13.458 -bool GCLabBitMapClosure::do_bit(size_t offset) { 13.459 - HeapWord* addr = _bitmap->offsetToHeapWord(offset); 13.460 - guarantee(_cm->isMarked(oop(addr)), "it should be!"); 13.461 - return true; 13.462 -} 13.463 -#endif // PRODUCT 13.464 - 13.465 G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size) : 13.466 - ParGCAllocBuffer(gclab_word_size), 13.467 - _should_mark_objects(false), 13.468 - _bitmap(G1CollectedHeap::heap()->reserved_region().start(), gclab_word_size), 13.469 - _retired(false) 13.470 -{ 13.471 - //_should_mark_objects is set to true when G1ParCopyHelper needs to 13.472 - // mark the forwarded location of an evacuated object. 13.473 - // We set _should_mark_objects to true if marking is active, i.e. when we 13.474 - // need to propagate a mark, or during an initial mark pause, i.e. when we 13.475 - // need to mark objects immediately reachable by the roots. 13.476 - if (G1CollectedHeap::heap()->mark_in_progress() || 13.477 - G1CollectedHeap::heap()->g1_policy()->during_initial_mark_pause()) { 13.478 - _should_mark_objects = true; 13.479 - } 13.480 -} 13.481 + ParGCAllocBuffer(gclab_word_size), _retired(false) { } 13.482 13.483 G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num) 13.484 : _g1h(g1h), 13.485 @@ -4323,8 +4174,7 @@ 13.486 _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured)), 13.487 _age_table(false), 13.488 _strong_roots_time(0), _term_time(0), 13.489 - _alloc_buffer_waste(0), _undo_waste(0) 13.490 -{ 13.491 + _alloc_buffer_waste(0), _undo_waste(0) { 13.492 // we allocate G1YoungSurvRateNumRegions plus one entries, since 13.493 // we "sacrifice" entry 0 to keep track of surviving bytes for 13.494 // non-young regions (where the age is -1) 13.495 @@ -4429,35 +4279,53 @@ 13.496 } while (!refs()->is_empty()); 13.497 } 13.498 13.499 -G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : 13.500 +G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, 13.501 + G1ParScanThreadState* par_scan_state) : 13.502 _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()), 13.503 _par_scan_state(par_scan_state), 13.504 _during_initial_mark(_g1->g1_policy()->during_initial_mark_pause()), 13.505 _mark_in_progress(_g1->mark_in_progress()) { } 13.506 13.507 -template <class T> void G1ParCopyHelper::mark_object(T* p) { 13.508 - // This is called from do_oop_work for objects that are not 13.509 - // in the collection set. Objects in the collection set 13.510 - // are marked after they have been evacuated. 13.511 - 13.512 - T heap_oop = oopDesc::load_heap_oop(p); 13.513 - if (!oopDesc::is_null(heap_oop)) { 13.514 - oop obj = oopDesc::decode_heap_oop(heap_oop); 13.515 - HeapWord* addr = (HeapWord*)obj; 13.516 - if (_g1->is_in_g1_reserved(addr)) { 13.517 - _cm->grayRoot(oop(addr)); 13.518 - } 13.519 - } 13.520 -} 13.521 - 13.522 -oop G1ParCopyHelper::copy_to_survivor_space(oop old, bool should_mark_root, 13.523 - bool should_mark_copy) { 13.524 +void G1ParCopyHelper::mark_object(oop obj) { 13.525 +#ifdef ASSERT 13.526 + HeapRegion* hr = _g1->heap_region_containing(obj); 13.527 + assert(hr != NULL, "sanity"); 13.528 + assert(!hr->in_collection_set(), "should not mark objects in the CSet"); 13.529 +#endif // ASSERT 13.530 + 13.531 + // We know that the object is not moving so it's safe to read its size. 13.532 + _cm->grayRoot(obj, (size_t) obj->size()); 13.533 +} 13.534 + 13.535 +void G1ParCopyHelper::mark_forwarded_object(oop from_obj, oop to_obj) { 13.536 +#ifdef ASSERT 13.537 + assert(from_obj->is_forwarded(), "from obj should be forwarded"); 13.538 + assert(from_obj->forwardee() == to_obj, "to obj should be the forwardee"); 13.539 + assert(from_obj != to_obj, "should not be self-forwarded"); 13.540 + 13.541 + HeapRegion* from_hr = _g1->heap_region_containing(from_obj); 13.542 + assert(from_hr != NULL, "sanity"); 13.543 + assert(from_hr->in_collection_set(), "from obj should be in the CSet"); 13.544 + 13.545 + HeapRegion* to_hr = _g1->heap_region_containing(to_obj); 13.546 + assert(to_hr != NULL, "sanity"); 13.547 + assert(!to_hr->in_collection_set(), "should not mark objects in the CSet"); 13.548 +#endif // ASSERT 13.549 + 13.550 + // The object might be in the process of being copied by another 13.551 + // worker so we cannot trust that its to-space image is 13.552 + // well-formed. So we have to read its size from its from-space 13.553 + // image which we know should not be changing. 13.554 + _cm->grayRoot(to_obj, (size_t) from_obj->size()); 13.555 +} 13.556 + 13.557 +oop G1ParCopyHelper::copy_to_survivor_space(oop old) { 13.558 size_t word_sz = old->size(); 13.559 HeapRegion* from_region = _g1->heap_region_containing_raw(old); 13.560 // +1 to make the -1 indexes valid... 13.561 int young_index = from_region->young_index_in_cset()+1; 13.562 - assert( (from_region->is_young() && young_index > 0) || 13.563 - (!from_region->is_young() && young_index == 0), "invariant" ); 13.564 + assert( (from_region->is_young() && young_index > 0) || 13.565 + (!from_region->is_young() && young_index == 0), "invariant" ); 13.566 G1CollectorPolicy* g1p = _g1->g1_policy(); 13.567 markOop m = old->mark(); 13.568 int age = m->has_displaced_mark_helper() ? m->displaced_mark_helper()->age() 13.569 @@ -4471,7 +4339,7 @@ 13.570 // This will either forward-to-self, or detect that someone else has 13.571 // installed a forwarding pointer. 13.572 OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure(); 13.573 - return _g1->handle_evacuation_failure_par(cl, old, should_mark_root); 13.574 + return _g1->handle_evacuation_failure_par(cl, old); 13.575 } 13.576 13.577 // We're going to allocate linearly, so might as well prefetch ahead. 13.578 @@ -4507,28 +4375,14 @@ 13.579 obj->set_mark(m); 13.580 } 13.581 13.582 - // Mark the evacuated object or propagate "next" mark bit 13.583 - if (should_mark_copy) { 13.584 - if (!use_local_bitmaps || 13.585 - !_par_scan_state->alloc_buffer(alloc_purpose)->mark(obj_ptr)) { 13.586 - // if we couldn't mark it on the local bitmap (this happens when 13.587 - // the object was not allocated in the GCLab), we have to bite 13.588 - // the bullet and do the standard parallel mark 13.589 - _cm->markAndGrayObjectIfNecessary(obj); 13.590 - } 13.591 - 13.592 - if (_g1->isMarkedNext(old)) { 13.593 - // Unmark the object's old location so that marking 13.594 - // doesn't think the old object is alive. 13.595 - _cm->nextMarkBitMap()->parClear((HeapWord*)old); 13.596 - } 13.597 - } 13.598 - 13.599 size_t* surv_young_words = _par_scan_state->surviving_young_words(); 13.600 surv_young_words[young_index] += word_sz; 13.601 13.602 if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) { 13.603 - arrayOop(old)->set_length(0); 13.604 + // We keep track of the next start index in the length field of 13.605 + // the to-space object. The actual length can be found in the 13.606 + // length field of the from-space object. 13.607 + arrayOop(obj)->set_length(0); 13.608 oop* old_p = set_partial_array_mask(old); 13.609 _par_scan_state->push_on_queue(old_p); 13.610 } else { 13.611 @@ -4550,61 +4404,24 @@ 13.612 ::do_oop_work(T* p) { 13.613 oop obj = oopDesc::load_decode_heap_oop(p); 13.614 assert(barrier != G1BarrierRS || obj != NULL, 13.615 - "Precondition: G1BarrierRS implies obj is nonNull"); 13.616 - 13.617 - // Marking: 13.618 - // If the object is in the collection set, then the thread 13.619 - // that copies the object should mark, or propagate the 13.620 - // mark to, the evacuated object. 13.621 - // If the object is not in the collection set then we 13.622 - // should call the mark_object() method depending on the 13.623 - // value of the template parameter do_mark_object (which will 13.624 - // be true for root scanning closures during an initial mark 13.625 - // pause). 13.626 - // The mark_object() method first checks whether the object 13.627 - // is marked and, if not, attempts to mark the object. 13.628 + "Precondition: G1BarrierRS implies obj is non-NULL"); 13.629 13.630 // here the null check is implicit in the cset_fast_test() test 13.631 if (_g1->in_cset_fast_test(obj)) { 13.632 + oop forwardee; 13.633 if (obj->is_forwarded()) { 13.634 - oopDesc::encode_store_heap_oop(p, obj->forwardee()); 13.635 - // If we are a root scanning closure during an initial 13.636 - // mark pause (i.e. do_mark_object will be true) then 13.637 - // we also need to handle marking of roots in the 13.638 - // event of an evacuation failure. In the event of an 13.639 - // evacuation failure, the object is forwarded to itself 13.640 - // and not copied. For root-scanning closures, the 13.641 - // object would be marked after a successful self-forward 13.642 - // but an object could be pointed to by both a root and non 13.643 - // root location and be self-forwarded by a non-root-scanning 13.644 - // closure. Therefore we also have to attempt to mark the 13.645 - // self-forwarded root object here. 13.646 - if (do_mark_object && obj->forwardee() == obj) { 13.647 - mark_object(p); 13.648 - } 13.649 + forwardee = obj->forwardee(); 13.650 } else { 13.651 - // During an initial mark pause, objects that are pointed to 13.652 - // by the roots need to be marked - even in the event of an 13.653 - // evacuation failure. We pass the template parameter 13.654 - // do_mark_object (which is true for root scanning closures 13.655 - // during an initial mark pause) to copy_to_survivor_space 13.656 - // which will pass it on to the evacuation failure handling 13.657 - // code. The thread that successfully self-forwards a root 13.658 - // object to itself is responsible for marking the object. 13.659 - bool should_mark_root = do_mark_object; 13.660 - 13.661 - // We need to mark the copied object if we're a root scanning 13.662 - // closure during an initial mark pause (i.e. do_mark_object 13.663 - // will be true), or the object is already marked and we need 13.664 - // to propagate the mark to the evacuated copy. 13.665 - bool should_mark_copy = do_mark_object || 13.666 - _during_initial_mark || 13.667 - (_mark_in_progress && !_g1->is_obj_ill(obj)); 13.668 - 13.669 - oop copy_oop = copy_to_survivor_space(obj, should_mark_root, 13.670 - should_mark_copy); 13.671 - oopDesc::encode_store_heap_oop(p, copy_oop); 13.672 + forwardee = copy_to_survivor_space(obj); 13.673 } 13.674 + assert(forwardee != NULL, "forwardee should not be NULL"); 13.675 + oopDesc::encode_store_heap_oop(p, forwardee); 13.676 + if (do_mark_object && forwardee != obj) { 13.677 + // If the object is self-forwarded we don't need to explicitly 13.678 + // mark it, the evacuation failure protocol will do so. 13.679 + mark_forwarded_object(obj, forwardee); 13.680 + } 13.681 + 13.682 // When scanning the RS, we only care about objs in CS. 13.683 if (barrier == G1BarrierRS) { 13.684 _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num()); 13.685 @@ -4613,8 +4430,8 @@ 13.686 // The object is not in collection set. If we're a root scanning 13.687 // closure during an initial mark pause (i.e. do_mark_object will 13.688 // be true) then attempt to mark the object. 13.689 - if (do_mark_object) { 13.690 - mark_object(p); 13.691 + if (do_mark_object && _g1->is_in_g1_reserved(obj)) { 13.692 + mark_object(obj); 13.693 } 13.694 } 13.695 13.696 @@ -4632,35 +4449,51 @@ 13.697 13.698 template <class T> void G1ParScanPartialArrayClosure::do_oop_nv(T* p) { 13.699 assert(has_partial_array_mask(p), "invariant"); 13.700 - oop old = clear_partial_array_mask(p); 13.701 - assert(old->is_objArray(), "must be obj array"); 13.702 - assert(old->is_forwarded(), "must be forwarded"); 13.703 - assert(Universe::heap()->is_in_reserved(old), "must be in heap."); 13.704 - 13.705 - objArrayOop obj = objArrayOop(old->forwardee()); 13.706 - assert((void*)old != (void*)old->forwardee(), "self forwarding here?"); 13.707 - // Process ParGCArrayScanChunk elements now 13.708 - // and push the remainder back onto queue 13.709 - int start = arrayOop(old)->length(); 13.710 - int end = obj->length(); 13.711 - int remainder = end - start; 13.712 - assert(start <= end, "just checking"); 13.713 + oop from_obj = clear_partial_array_mask(p); 13.714 + 13.715 + assert(Universe::heap()->is_in_reserved(from_obj), "must be in heap."); 13.716 + assert(from_obj->is_objArray(), "must be obj array"); 13.717 + objArrayOop from_obj_array = objArrayOop(from_obj); 13.718 + // The from-space object contains the real length. 13.719 + int length = from_obj_array->length(); 13.720 + 13.721 + assert(from_obj->is_forwarded(), "must be forwarded"); 13.722 + oop to_obj = from_obj->forwardee(); 13.723 + assert(from_obj != to_obj, "should not be chunking self-forwarded objects"); 13.724 + objArrayOop to_obj_array = objArrayOop(to_obj); 13.725 + // We keep track of the next start index in the length field of the 13.726 + // to-space object. 13.727 + int next_index = to_obj_array->length(); 13.728 + assert(0 <= next_index && next_index < length, 13.729 + err_msg("invariant, next index: %d, length: %d", next_index, length)); 13.730 + 13.731 + int start = next_index; 13.732 + int end = length; 13.733 + int remainder = end - start; 13.734 + // We'll try not to push a range that's smaller than ParGCArrayScanChunk. 13.735 if (remainder > 2 * ParGCArrayScanChunk) { 13.736 - // Test above combines last partial chunk with a full chunk 13.737 end = start + ParGCArrayScanChunk; 13.738 - arrayOop(old)->set_length(end); 13.739 - // Push remainder. 13.740 - oop* old_p = set_partial_array_mask(old); 13.741 - assert(arrayOop(old)->length() < obj->length(), "Empty push?"); 13.742 - _par_scan_state->push_on_queue(old_p); 13.743 + to_obj_array->set_length(end); 13.744 + // Push the remainder before we process the range in case another 13.745 + // worker has run out of things to do and can steal it. 13.746 + oop* from_obj_p = set_partial_array_mask(from_obj); 13.747 + _par_scan_state->push_on_queue(from_obj_p); 13.748 } else { 13.749 - // Restore length so that the heap remains parsable in 13.750 - // case of evacuation failure. 13.751 - arrayOop(old)->set_length(end); 13.752 - } 13.753 - _scanner.set_region(_g1->heap_region_containing_raw(obj)); 13.754 - // process our set of indices (include header in first chunk) 13.755 - obj->oop_iterate_range(&_scanner, start, end); 13.756 + assert(length == end, "sanity"); 13.757 + // We'll process the final range for this object. Restore the length 13.758 + // so that the heap remains parsable in case of evacuation failure. 13.759 + to_obj_array->set_length(end); 13.760 + } 13.761 + _scanner.set_region(_g1->heap_region_containing_raw(to_obj)); 13.762 + // Process indexes [start,end). It will also process the header 13.763 + // along with the first chunk (i.e., the chunk with start == 0). 13.764 + // Note that at this point the length field of to_obj_array is not 13.765 + // correct given that we are using it to keep track of the next 13.766 + // start index. oop_iterate_range() (thankfully!) ignores the length 13.767 + // field and only relies on the start / end parameters. It does 13.768 + // however return the size of the object which will be incorrect. So 13.769 + // we have to ignore it even if we wanted to use it. 13.770 + to_obj_array->oop_iterate_range(&_scanner, start, end); 13.771 } 13.772 13.773 class G1ParEvacuateFollowersClosure : public VoidClosure { 13.774 @@ -4893,12 +4726,16 @@ 13.775 13.776 g1_policy()->record_ext_root_scan_time(worker_i, ext_root_time_ms); 13.777 13.778 - // Scan strong roots in mark stack. 13.779 - if (!_process_strong_tasks->is_task_claimed(G1H_PS_mark_stack_oops_do)) { 13.780 - concurrent_mark()->oops_do(scan_non_heap_roots); 13.781 - } 13.782 - double mark_stack_scan_ms = (os::elapsedTime() - ext_roots_end) * 1000.0; 13.783 - g1_policy()->record_mark_stack_scan_time(worker_i, mark_stack_scan_ms); 13.784 + // During conc marking we have to filter the per-thread SATB buffers 13.785 + // to make sure we remove any oops into the CSet (which will show up 13.786 + // as implicitly live). 13.787 + if (!_process_strong_tasks->is_task_claimed(G1H_PS_filter_satb_buffers)) { 13.788 + if (mark_in_progress()) { 13.789 + JavaThread::satb_mark_queue_set().filter_thread_buffers(); 13.790 + } 13.791 + } 13.792 + double satb_filtering_ms = (os::elapsedTime() - ext_roots_end) * 1000.0; 13.793 + g1_policy()->record_satb_filtering_time(worker_i, satb_filtering_ms); 13.794 13.795 // Now scan the complement of the collection set. 13.796 if (scan_rs != NULL) { 13.797 @@ -5439,6 +5276,7 @@ 13.798 } 13.799 13.800 void G1CollectedHeap::evacuate_collection_set() { 13.801 + _expand_heap_after_alloc_failure = true; 13.802 set_evacuation_failed(false); 13.803 13.804 g1_rem_set()->prepare_for_oops_into_collection_set_do(); 13.805 @@ -5516,13 +5354,6 @@ 13.806 13.807 finalize_for_evac_failure(); 13.808 13.809 - // Must do this before clearing the per-region evac-failure flags 13.810 - // (which is currently done when we free the collection set). 13.811 - // We also only do this if marking is actually in progress and so 13.812 - // have to do this before we set the mark_in_progress flag at the 13.813 - // end of an initial mark pause. 13.814 - concurrent_mark()->complete_marking_in_collection_set(); 13.815 - 13.816 if (evacuation_failed()) { 13.817 remove_self_forwarding_pointers(); 13.818 if (PrintGCDetails) { 13.819 @@ -6179,6 +6010,8 @@ 13.820 } else { 13.821 _hr_printer.alloc(new_alloc_region, G1HRPrinter::Old); 13.822 } 13.823 + bool during_im = g1_policy()->during_initial_mark_pause(); 13.824 + new_alloc_region->note_start_of_copying(during_im); 13.825 return new_alloc_region; 13.826 } else { 13.827 g1_policy()->note_alloc_region_limit_reached(ap); 13.828 @@ -6190,7 +6023,8 @@ 13.829 void G1CollectedHeap::retire_gc_alloc_region(HeapRegion* alloc_region, 13.830 size_t allocated_bytes, 13.831 GCAllocPurpose ap) { 13.832 - alloc_region->note_end_of_copying(); 13.833 + bool during_im = g1_policy()->during_initial_mark_pause(); 13.834 + alloc_region->note_end_of_copying(during_im); 13.835 g1_policy()->record_bytes_copied_during_gc(allocated_bytes); 13.836 if (ap == GCAllocForSurvived) { 13.837 young_list()->add_survivor_region(alloc_region);
14.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Tue Jan 17 13:08:52 2012 -0500 14.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Tue Jan 17 21:25:28 2012 -0500 14.3 @@ -1,5 +1,5 @@ 14.4 /* 14.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 14.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 14.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 14.8 * 14.9 * This code is free software; you can redistribute it and/or modify it 14.10 @@ -285,6 +285,14 @@ 14.11 // Typically, it is not full so we should re-use it during the next GC. 14.12 HeapRegion* _retained_old_gc_alloc_region; 14.13 14.14 + // It specifies whether we should attempt to expand the heap after a 14.15 + // region allocation failure. If heap expansion fails we set this to 14.16 + // false so that we don't re-attempt the heap expansion (it's likely 14.17 + // that subsequent expansion attempts will also fail if one fails). 14.18 + // Currently, it is only consulted during GC and it's reset at the 14.19 + // start of each GC. 14.20 + bool _expand_heap_after_alloc_failure; 14.21 + 14.22 // It resets the mutator alloc region before new allocations can take place. 14.23 void init_mutator_alloc_region(); 14.24 14.25 @@ -861,8 +869,7 @@ 14.26 void finalize_for_evac_failure(); 14.27 14.28 // An attempt to evacuate "obj" has failed; take necessary steps. 14.29 - oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj, 14.30 - bool should_mark_root); 14.31 + oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj); 14.32 void handle_evacuation_failure_common(oop obj, markOop m); 14.33 14.34 // ("Weak") Reference processing support. 14.35 @@ -954,7 +961,7 @@ 14.36 unsigned int* _worker_cset_start_region_time_stamp; 14.37 14.38 enum G1H_process_strong_roots_tasks { 14.39 - G1H_PS_mark_stack_oops_do, 14.40 + G1H_PS_filter_satb_buffers, 14.41 G1H_PS_refProcessor_oops_do, 14.42 // Leave this one last. 14.43 G1H_PS_NumElements 14.44 @@ -1305,6 +1312,10 @@ 14.45 // It resets all the region claim values to the default. 14.46 void reset_heap_region_claim_values(); 14.47 14.48 + // Resets the claim values of regions in the current 14.49 + // collection set to the default. 14.50 + void reset_cset_heap_region_claim_values(); 14.51 + 14.52 #ifdef ASSERT 14.53 bool check_heap_region_claim_values(jint claim_value); 14.54 14.55 @@ -1740,10 +1751,8 @@ 14.56 _gclab_word_size(gclab_word_size), 14.57 _real_start_word(NULL), 14.58 _real_end_word(NULL), 14.59 - _start_word(NULL) 14.60 - { 14.61 - guarantee( size_in_words() >= bitmap_size_in_words(), 14.62 - "just making sure"); 14.63 + _start_word(NULL) { 14.64 + guarantee(false, "GCLabBitMap::GCLabBitmap(): don't call this any more"); 14.65 } 14.66 14.67 inline unsigned heapWordToOffset(HeapWord* addr) { 14.68 @@ -1797,6 +1806,8 @@ 14.69 } 14.70 14.71 void set_buffer(HeapWord* start) { 14.72 + guarantee(false, "set_buffer(): don't call this any more"); 14.73 + 14.74 guarantee(use_local_bitmaps, "invariant"); 14.75 clear(); 14.76 14.77 @@ -1820,6 +1831,8 @@ 14.78 #endif // PRODUCT 14.79 14.80 void retire() { 14.81 + guarantee(false, "retire(): don't call this any more"); 14.82 + 14.83 guarantee(use_local_bitmaps, "invariant"); 14.84 assert(fields_well_formed(), "invariant"); 14.85 14.86 @@ -1853,32 +1866,18 @@ 14.87 class G1ParGCAllocBuffer: public ParGCAllocBuffer { 14.88 private: 14.89 bool _retired; 14.90 - bool _should_mark_objects; 14.91 - GCLabBitMap _bitmap; 14.92 14.93 public: 14.94 G1ParGCAllocBuffer(size_t gclab_word_size); 14.95 14.96 - inline bool mark(HeapWord* addr) { 14.97 - guarantee(use_local_bitmaps, "invariant"); 14.98 - assert(_should_mark_objects, "invariant"); 14.99 - return _bitmap.mark(addr); 14.100 - } 14.101 - 14.102 - inline void set_buf(HeapWord* buf) { 14.103 - if (use_local_bitmaps && _should_mark_objects) { 14.104 - _bitmap.set_buffer(buf); 14.105 - } 14.106 + void set_buf(HeapWord* buf) { 14.107 ParGCAllocBuffer::set_buf(buf); 14.108 _retired = false; 14.109 } 14.110 14.111 - inline void retire(bool end_of_gc, bool retain) { 14.112 + void retire(bool end_of_gc, bool retain) { 14.113 if (_retired) 14.114 return; 14.115 - if (use_local_bitmaps && _should_mark_objects) { 14.116 - _bitmap.retire(); 14.117 - } 14.118 ParGCAllocBuffer::retire(end_of_gc, retain); 14.119 _retired = true; 14.120 }
15.1 --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Tue Jan 17 13:08:52 2012 -0500 15.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Tue Jan 17 21:25:28 2012 -0500 15.3 @@ -1,5 +1,5 @@ 15.4 /* 15.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 15.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 15.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 15.8 * 15.9 * This code is free software; you can redistribute it and/or modify it 15.10 @@ -281,7 +281,7 @@ 15.11 15.12 _par_last_gc_worker_start_times_ms = new double[_parallel_gc_threads]; 15.13 _par_last_ext_root_scan_times_ms = new double[_parallel_gc_threads]; 15.14 - _par_last_mark_stack_scan_times_ms = new double[_parallel_gc_threads]; 15.15 + _par_last_satb_filtering_times_ms = new double[_parallel_gc_threads]; 15.16 15.17 _par_last_update_rs_times_ms = new double[_parallel_gc_threads]; 15.18 _par_last_update_rs_processed_buffers = new double[_parallel_gc_threads]; 15.19 @@ -905,10 +905,19 @@ 15.20 gclog_or_tty->print(" (%s)", gcs_are_young() ? "young" : "mixed"); 15.21 } 15.22 15.23 - // We only need to do this here as the policy will only be applied 15.24 - // to the GC we're about to start. so, no point is calculating this 15.25 - // every time we calculate / recalculate the target young length. 15.26 - update_survivors_policy(); 15.27 + if (!during_initial_mark_pause()) { 15.28 + // We only need to do this here as the policy will only be applied 15.29 + // to the GC we're about to start. so, no point is calculating this 15.30 + // every time we calculate / recalculate the target young length. 15.31 + update_survivors_policy(); 15.32 + } else { 15.33 + // The marking phase has a "we only copy implicitly live 15.34 + // objects during marking" invariant. The easiest way to ensure it 15.35 + // holds is not to allocate any survivor regions and tenure all 15.36 + // objects. In the future we might change this and handle survivor 15.37 + // regions specially during marking. 15.38 + tenure_all_objects(); 15.39 + } 15.40 15.41 assert(_g1->used() == _g1->recalculate_used(), 15.42 err_msg("sanity, used: "SIZE_FORMAT" recalculate_used: "SIZE_FORMAT, 15.43 @@ -939,7 +948,7 @@ 15.44 for (int i = 0; i < _parallel_gc_threads; ++i) { 15.45 _par_last_gc_worker_start_times_ms[i] = -1234.0; 15.46 _par_last_ext_root_scan_times_ms[i] = -1234.0; 15.47 - _par_last_mark_stack_scan_times_ms[i] = -1234.0; 15.48 + _par_last_satb_filtering_times_ms[i] = -1234.0; 15.49 _par_last_update_rs_times_ms[i] = -1234.0; 15.50 _par_last_update_rs_processed_buffers[i] = -1234.0; 15.51 _par_last_scan_rs_times_ms[i] = -1234.0; 15.52 @@ -1227,7 +1236,7 @@ 15.53 // of the PrintGCDetails output, in the non-parallel case. 15.54 15.55 double ext_root_scan_time = avg_value(_par_last_ext_root_scan_times_ms); 15.56 - double mark_stack_scan_time = avg_value(_par_last_mark_stack_scan_times_ms); 15.57 + double satb_filtering_time = avg_value(_par_last_satb_filtering_times_ms); 15.58 double update_rs_time = avg_value(_par_last_update_rs_times_ms); 15.59 double update_rs_processed_buffers = 15.60 sum_of_values(_par_last_update_rs_processed_buffers); 15.61 @@ -1236,7 +1245,7 @@ 15.62 double termination_time = avg_value(_par_last_termination_times_ms); 15.63 15.64 double known_time = ext_root_scan_time + 15.65 - mark_stack_scan_time + 15.66 + satb_filtering_time + 15.67 update_rs_time + 15.68 scan_rs_time + 15.69 obj_copy_time; 15.70 @@ -1282,7 +1291,7 @@ 15.71 body_summary->record_satb_drain_time_ms(_cur_satb_drain_time_ms); 15.72 15.73 body_summary->record_ext_root_scan_time_ms(ext_root_scan_time); 15.74 - body_summary->record_mark_stack_scan_time_ms(mark_stack_scan_time); 15.75 + body_summary->record_satb_filtering_time_ms(satb_filtering_time); 15.76 body_summary->record_update_rs_time_ms(update_rs_time); 15.77 body_summary->record_scan_rs_time_ms(scan_rs_time); 15.78 body_summary->record_obj_copy_time_ms(obj_copy_time); 15.79 @@ -1376,16 +1385,12 @@ 15.80 (last_pause_included_initial_mark) ? " (initial-mark)" : "", 15.81 elapsed_ms / 1000.0); 15.82 15.83 - if (print_marking_info) { 15.84 - print_stats(1, "SATB Drain Time", _cur_satb_drain_time_ms); 15.85 - } 15.86 - 15.87 if (parallel) { 15.88 print_stats(1, "Parallel Time", _cur_collection_par_time_ms); 15.89 print_par_stats(2, "GC Worker Start", _par_last_gc_worker_start_times_ms); 15.90 print_par_stats(2, "Ext Root Scanning", _par_last_ext_root_scan_times_ms); 15.91 if (print_marking_info) { 15.92 - print_par_stats(2, "Mark Stack Scanning", _par_last_mark_stack_scan_times_ms); 15.93 + print_par_stats(2, "SATB Filtering", _par_last_satb_filtering_times_ms); 15.94 } 15.95 print_par_stats(2, "Update RS", _par_last_update_rs_times_ms); 15.96 print_par_sizes(3, "Processed Buffers", _par_last_update_rs_processed_buffers); 15.97 @@ -1399,7 +1404,7 @@ 15.98 _par_last_gc_worker_times_ms[i] = _par_last_gc_worker_end_times_ms[i] - _par_last_gc_worker_start_times_ms[i]; 15.99 15.100 double worker_known_time = _par_last_ext_root_scan_times_ms[i] + 15.101 - _par_last_mark_stack_scan_times_ms[i] + 15.102 + _par_last_satb_filtering_times_ms[i] + 15.103 _par_last_update_rs_times_ms[i] + 15.104 _par_last_scan_rs_times_ms[i] + 15.105 _par_last_obj_copy_times_ms[i] + 15.106 @@ -1412,7 +1417,7 @@ 15.107 } else { 15.108 print_stats(1, "Ext Root Scanning", ext_root_scan_time); 15.109 if (print_marking_info) { 15.110 - print_stats(1, "Mark Stack Scanning", mark_stack_scan_time); 15.111 + print_stats(1, "SATB Filtering", satb_filtering_time); 15.112 } 15.113 print_stats(1, "Update RS", update_rs_time); 15.114 print_stats(2, "Processed Buffers", (int)update_rs_processed_buffers); 15.115 @@ -1983,11 +1988,10 @@ 15.116 if (summary->get_total_seq()->num() > 0) { 15.117 print_summary_sd(0, "Evacuation Pauses", summary->get_total_seq()); 15.118 if (body_summary != NULL) { 15.119 - print_summary(1, "SATB Drain", body_summary->get_satb_drain_seq()); 15.120 if (parallel) { 15.121 print_summary(1, "Parallel Time", body_summary->get_parallel_seq()); 15.122 print_summary(2, "Ext Root Scanning", body_summary->get_ext_root_scan_seq()); 15.123 - print_summary(2, "Mark Stack Scanning", body_summary->get_mark_stack_scan_seq()); 15.124 + print_summary(2, "SATB Filtering", body_summary->get_satb_filtering_seq()); 15.125 print_summary(2, "Update RS", body_summary->get_update_rs_seq()); 15.126 print_summary(2, "Scan RS", body_summary->get_scan_rs_seq()); 15.127 print_summary(2, "Object Copy", body_summary->get_obj_copy_seq()); 15.128 @@ -1996,7 +2000,7 @@ 15.129 { 15.130 NumberSeq* other_parts[] = { 15.131 body_summary->get_ext_root_scan_seq(), 15.132 - body_summary->get_mark_stack_scan_seq(), 15.133 + body_summary->get_satb_filtering_seq(), 15.134 body_summary->get_update_rs_seq(), 15.135 body_summary->get_scan_rs_seq(), 15.136 body_summary->get_obj_copy_seq(), 15.137 @@ -2009,7 +2013,7 @@ 15.138 } 15.139 } else { 15.140 print_summary(1, "Ext Root Scanning", body_summary->get_ext_root_scan_seq()); 15.141 - print_summary(1, "Mark Stack Scanning", body_summary->get_mark_stack_scan_seq()); 15.142 + print_summary(1, "SATB Filtering", body_summary->get_satb_filtering_seq()); 15.143 print_summary(1, "Update RS", body_summary->get_update_rs_seq()); 15.144 print_summary(1, "Scan RS", body_summary->get_scan_rs_seq()); 15.145 print_summary(1, "Object Copy", body_summary->get_obj_copy_seq()); 15.146 @@ -2036,7 +2040,7 @@ 15.147 body_summary->get_satb_drain_seq(), 15.148 body_summary->get_update_rs_seq(), 15.149 body_summary->get_ext_root_scan_seq(), 15.150 - body_summary->get_mark_stack_scan_seq(), 15.151 + body_summary->get_satb_filtering_seq(), 15.152 body_summary->get_scan_rs_seq(), 15.153 body_summary->get_obj_copy_seq() 15.154 }; 15.155 @@ -2433,9 +2437,6 @@ 15.156 assert(_inc_cset_build_state == Active, "Precondition"); 15.157 assert(!hr->is_young(), "non-incremental add of young region"); 15.158 15.159 - if (_g1->mark_in_progress()) 15.160 - _g1->concurrent_mark()->registerCSetRegion(hr); 15.161 - 15.162 assert(!hr->in_collection_set(), "should not already be in the CSet"); 15.163 hr->set_in_collection_set(true); 15.164 hr->set_next_in_collection_set(_collection_set); 15.165 @@ -2705,9 +2706,6 @@ 15.166 // Clear the fields that point to the survivor list - they are all young now. 15.167 young_list->clear_survivors(); 15.168 15.169 - if (_g1->mark_in_progress()) 15.170 - _g1->concurrent_mark()->register_collection_set_finger(_inc_cset_max_finger); 15.171 - 15.172 _collection_set = _inc_cset_head; 15.173 _collection_set_bytes_used_before = _inc_cset_bytes_used_before; 15.174 time_remaining_ms -= _inc_cset_predicted_elapsed_time_ms;
16.1 --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp Tue Jan 17 13:08:52 2012 -0500 16.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp Tue Jan 17 21:25:28 2012 -0500 16.3 @@ -1,5 +1,5 @@ 16.4 /* 16.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 16.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 16.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 16.8 * 16.9 * This code is free software; you can redistribute it and/or modify it 16.10 @@ -67,7 +67,7 @@ 16.11 define_num_seq(satb_drain) // optional 16.12 define_num_seq(parallel) // parallel only 16.13 define_num_seq(ext_root_scan) 16.14 - define_num_seq(mark_stack_scan) 16.15 + define_num_seq(satb_filtering) 16.16 define_num_seq(update_rs) 16.17 define_num_seq(scan_rs) 16.18 define_num_seq(obj_copy) 16.19 @@ -215,7 +215,7 @@ 16.20 16.21 double* _par_last_gc_worker_start_times_ms; 16.22 double* _par_last_ext_root_scan_times_ms; 16.23 - double* _par_last_mark_stack_scan_times_ms; 16.24 + double* _par_last_satb_filtering_times_ms; 16.25 double* _par_last_update_rs_times_ms; 16.26 double* _par_last_update_rs_processed_buffers; 16.27 double* _par_last_scan_rs_times_ms; 16.28 @@ -841,8 +841,8 @@ 16.29 _par_last_ext_root_scan_times_ms[worker_i] = ms; 16.30 } 16.31 16.32 - void record_mark_stack_scan_time(int worker_i, double ms) { 16.33 - _par_last_mark_stack_scan_times_ms[worker_i] = ms; 16.34 + void record_satb_filtering_time(int worker_i, double ms) { 16.35 + _par_last_satb_filtering_times_ms[worker_i] = ms; 16.36 } 16.37 16.38 void record_satb_drain_time(double ms) { 16.39 @@ -1146,6 +1146,11 @@ 16.40 _survivor_surv_rate_group->stop_adding_regions(); 16.41 } 16.42 16.43 + void tenure_all_objects() { 16.44 + _max_survivor_regions = 0; 16.45 + _tenuring_threshold = 0; 16.46 + } 16.47 + 16.48 void record_survivor_regions(size_t regions, 16.49 HeapRegion* head, 16.50 HeapRegion* tail) {
17.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 17.2 +++ b/src/share/vm/gc_implementation/g1/g1EvacFailure.hpp Tue Jan 17 21:25:28 2012 -0500 17.3 @@ -0,0 +1,236 @@ 17.4 +/* 17.5 + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. 17.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 17.7 + * 17.8 + * This code is free software; you can redistribute it and/or modify it 17.9 + * under the terms of the GNU General Public License version 2 only, as 17.10 + * published by the Free Software Foundation. 17.11 + * 17.12 + * This code is distributed in the hope that it will be useful, but WITHOUT 17.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 17.14 + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 17.15 + * version 2 for more details (a copy is included in the LICENSE file that 17.16 + * accompanied this code). 17.17 + * 17.18 + * You should have received a copy of the GNU General Public License version 17.19 + * 2 along with this work; if not, write to the Free Software Foundation, 17.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17.21 + * 17.22 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 17.23 + * or visit www.oracle.com if you need additional information or have any 17.24 + * questions. 17.25 + * 17.26 + */ 17.27 + 17.28 +#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP 17.29 +#define SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP 17.30 + 17.31 +#include "gc_implementation/g1/concurrentMark.inline.hpp" 17.32 +#include "gc_implementation/g1/dirtyCardQueue.hpp" 17.33 +#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" 17.34 +#include "gc_implementation/g1/g1_globals.hpp" 17.35 +#include "gc_implementation/g1/g1OopClosures.inline.hpp" 17.36 +#include "gc_implementation/g1/heapRegion.hpp" 17.37 +#include "gc_implementation/g1/heapRegionRemSet.hpp" 17.38 +#include "utilities/workgroup.hpp" 17.39 + 17.40 +// Closures and tasks associated with any self-forwarding pointers 17.41 +// installed as a result of an evacuation failure. 17.42 + 17.43 +class UpdateRSetDeferred : public OopsInHeapRegionClosure { 17.44 +private: 17.45 + G1CollectedHeap* _g1; 17.46 + DirtyCardQueue *_dcq; 17.47 + CardTableModRefBS* _ct_bs; 17.48 + 17.49 +public: 17.50 + UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) : 17.51 + _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {} 17.52 + 17.53 + virtual void do_oop(narrowOop* p) { do_oop_work(p); } 17.54 + virtual void do_oop( oop* p) { do_oop_work(p); } 17.55 + template <class T> void do_oop_work(T* p) { 17.56 + assert(_from->is_in_reserved(p), "paranoia"); 17.57 + if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && 17.58 + !_from->is_survivor()) { 17.59 + size_t card_index = _ct_bs->index_for(p); 17.60 + if (_ct_bs->mark_card_deferred(card_index)) { 17.61 + _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index)); 17.62 + } 17.63 + } 17.64 + } 17.65 +}; 17.66 + 17.67 +class RemoveSelfForwardPtrObjClosure: public ObjectClosure { 17.68 +private: 17.69 + G1CollectedHeap* _g1; 17.70 + ConcurrentMark* _cm; 17.71 + HeapRegion* _hr; 17.72 + size_t _marked_bytes; 17.73 + OopsInHeapRegionClosure *_update_rset_cl; 17.74 + bool _during_initial_mark; 17.75 + bool _during_conc_mark; 17.76 +public: 17.77 + RemoveSelfForwardPtrObjClosure(G1CollectedHeap* g1, ConcurrentMark* cm, 17.78 + HeapRegion* hr, 17.79 + OopsInHeapRegionClosure* update_rset_cl, 17.80 + bool during_initial_mark, 17.81 + bool during_conc_mark) : 17.82 + _g1(g1), _cm(cm), _hr(hr), _marked_bytes(0), 17.83 + _update_rset_cl(update_rset_cl), 17.84 + _during_initial_mark(during_initial_mark), 17.85 + _during_conc_mark(during_conc_mark) { } 17.86 + 17.87 + size_t marked_bytes() { return _marked_bytes; } 17.88 + 17.89 + // <original comment> 17.90 + // The original idea here was to coalesce evacuated and dead objects. 17.91 + // However that caused complications with the block offset table (BOT). 17.92 + // In particular if there were two TLABs, one of them partially refined. 17.93 + // |----- TLAB_1--------|----TLAB_2-~~~(partially refined part)~~~| 17.94 + // The BOT entries of the unrefined part of TLAB_2 point to the start 17.95 + // of TLAB_2. If the last object of the TLAB_1 and the first object 17.96 + // of TLAB_2 are coalesced, then the cards of the unrefined part 17.97 + // would point into middle of the filler object. 17.98 + // The current approach is to not coalesce and leave the BOT contents intact. 17.99 + // </original comment> 17.100 + // 17.101 + // We now reset the BOT when we start the object iteration over the 17.102 + // region and refine its entries for every object we come across. So 17.103 + // the above comment is not really relevant and we should be able 17.104 + // to coalesce dead objects if we want to. 17.105 + void do_object(oop obj) { 17.106 + HeapWord* obj_addr = (HeapWord*) obj; 17.107 + assert(_hr->is_in(obj_addr), "sanity"); 17.108 + size_t obj_size = obj->size(); 17.109 + _hr->update_bot_for_object(obj_addr, obj_size); 17.110 + 17.111 + if (obj->is_forwarded() && obj->forwardee() == obj) { 17.112 + // The object failed to move. 17.113 + 17.114 + // We consider all objects that we find self-forwarded to be 17.115 + // live. What we'll do is that we'll update the prev marking 17.116 + // info so that they are all under PTAMS and explicitly marked. 17.117 + _cm->markPrev(obj); 17.118 + if (_during_initial_mark) { 17.119 + // For the next marking info we'll only mark the 17.120 + // self-forwarded objects explicitly if we are during 17.121 + // initial-mark (since, normally, we only mark objects pointed 17.122 + // to by roots if we succeed in copying them). By marking all 17.123 + // self-forwarded objects we ensure that we mark any that are 17.124 + // still pointed to be roots. During concurrent marking, and 17.125 + // after initial-mark, we don't need to mark any objects 17.126 + // explicitly and all objects in the CSet are considered 17.127 + // (implicitly) live. So, we won't mark them explicitly and 17.128 + // we'll leave them over NTAMS. 17.129 + _cm->markNext(obj); 17.130 + } 17.131 + _marked_bytes += (obj_size * HeapWordSize); 17.132 + obj->set_mark(markOopDesc::prototype()); 17.133 + 17.134 + // While we were processing RSet buffers during the collection, 17.135 + // we actually didn't scan any cards on the collection set, 17.136 + // since we didn't want to update remembered sets with entries 17.137 + // that point into the collection set, given that live objects 17.138 + // from the collection set are about to move and such entries 17.139 + // will be stale very soon. 17.140 + // This change also dealt with a reliability issue which 17.141 + // involved scanning a card in the collection set and coming 17.142 + // across an array that was being chunked and looking malformed. 17.143 + // The problem is that, if evacuation fails, we might have 17.144 + // remembered set entries missing given that we skipped cards on 17.145 + // the collection set. So, we'll recreate such entries now. 17.146 + obj->oop_iterate(_update_rset_cl); 17.147 + assert(_cm->isPrevMarked(obj), "Should be marked!"); 17.148 + } else { 17.149 + // The object has been either evacuated or is dead. Fill it with a 17.150 + // dummy object. 17.151 + MemRegion mr((HeapWord*) obj, obj_size); 17.152 + CollectedHeap::fill_with_object(mr); 17.153 + } 17.154 + } 17.155 +}; 17.156 + 17.157 +class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure { 17.158 + G1CollectedHeap* _g1h; 17.159 + ConcurrentMark* _cm; 17.160 + OopsInHeapRegionClosure *_update_rset_cl; 17.161 + 17.162 +public: 17.163 + RemoveSelfForwardPtrHRClosure(G1CollectedHeap* g1h, 17.164 + OopsInHeapRegionClosure* update_rset_cl) : 17.165 + _g1h(g1h), _update_rset_cl(update_rset_cl), 17.166 + _cm(_g1h->concurrent_mark()) { } 17.167 + 17.168 + bool doHeapRegion(HeapRegion *hr) { 17.169 + bool during_initial_mark = _g1h->g1_policy()->during_initial_mark_pause(); 17.170 + bool during_conc_mark = _g1h->mark_in_progress(); 17.171 + 17.172 + assert(!hr->isHumongous(), "sanity"); 17.173 + assert(hr->in_collection_set(), "bad CS"); 17.174 + 17.175 + if (hr->claimHeapRegion(HeapRegion::ParEvacFailureClaimValue)) { 17.176 + if (hr->evacuation_failed()) { 17.177 + RemoveSelfForwardPtrObjClosure rspc(_g1h, _cm, hr, _update_rset_cl, 17.178 + during_initial_mark, 17.179 + during_conc_mark); 17.180 + 17.181 + MemRegion mr(hr->bottom(), hr->end()); 17.182 + // We'll recreate the prev marking info so we'll first clear 17.183 + // the prev bitmap range for this region. We never mark any 17.184 + // CSet objects explicitly so the next bitmap range should be 17.185 + // cleared anyway. 17.186 + _cm->clearRangePrevBitmap(mr); 17.187 + 17.188 + hr->note_self_forwarding_removal_start(during_initial_mark, 17.189 + during_conc_mark); 17.190 + 17.191 + // In the common case (i.e. when there is no evacuation 17.192 + // failure) we make sure that the following is done when 17.193 + // the region is freed so that it is "ready-to-go" when it's 17.194 + // re-allocated. However, when evacuation failure happens, a 17.195 + // region will remain in the heap and might ultimately be added 17.196 + // to a CSet in the future. So we have to be careful here and 17.197 + // make sure the region's RSet is ready for parallel iteration 17.198 + // whenever this might be required in the future. 17.199 + hr->rem_set()->reset_for_par_iteration(); 17.200 + hr->reset_bot(); 17.201 + _update_rset_cl->set_region(hr); 17.202 + hr->object_iterate(&rspc); 17.203 + 17.204 + hr->note_self_forwarding_removal_end(during_initial_mark, 17.205 + during_conc_mark, 17.206 + rspc.marked_bytes()); 17.207 + } 17.208 + } 17.209 + return false; 17.210 + } 17.211 +}; 17.212 + 17.213 +class G1ParRemoveSelfForwardPtrsTask: public AbstractGangTask { 17.214 +protected: 17.215 + G1CollectedHeap* _g1h; 17.216 + 17.217 +public: 17.218 + G1ParRemoveSelfForwardPtrsTask(G1CollectedHeap* g1h) : 17.219 + AbstractGangTask("G1 Remove Self-forwarding Pointers"), 17.220 + _g1h(g1h) { } 17.221 + 17.222 + void work(uint worker_id) { 17.223 + UpdateRSetImmediate immediate_update(_g1h->g1_rem_set()); 17.224 + DirtyCardQueue dcq(&_g1h->dirty_card_queue_set()); 17.225 + UpdateRSetDeferred deferred_update(_g1h, &dcq); 17.226 + 17.227 + OopsInHeapRegionClosure *update_rset_cl = &deferred_update; 17.228 + if (!G1DeferredRSUpdate) { 17.229 + update_rset_cl = &immediate_update; 17.230 + } 17.231 + 17.232 + RemoveSelfForwardPtrHRClosure rsfp_cl(_g1h, update_rset_cl); 17.233 + 17.234 + HeapRegion* hr = _g1h->start_cset_region_for_worker(worker_id); 17.235 + _g1h->collection_set_iterate_from(hr, &rsfp_cl); 17.236 + } 17.237 +}; 17.238 + 17.239 +#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP
18.1 --- a/src/share/vm/gc_implementation/g1/g1OopClosures.hpp Tue Jan 17 13:08:52 2012 -0500 18.2 +++ b/src/share/vm/gc_implementation/g1/g1OopClosures.hpp Tue Jan 17 21:25:28 2012 -0500 18.3 @@ -1,5 +1,5 @@ 18.4 /* 18.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 18.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 18.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 18.8 * 18.9 * This code is free software; you can redistribute it and/or modify it 18.10 @@ -121,17 +121,25 @@ 18.11 class G1ParCopyHelper : public G1ParClosureSuper { 18.12 G1ParScanClosure *_scanner; 18.13 protected: 18.14 - template <class T> void mark_object(T* p); 18.15 - oop copy_to_survivor_space(oop obj, bool should_mark_root, 18.16 - bool should_mark_copy); 18.17 + // Mark the object if it's not already marked. This is used to mark 18.18 + // objects pointed to by roots that are guaranteed not to move 18.19 + // during the GC (i.e., non-CSet objects). It is MT-safe. 18.20 + void mark_object(oop obj); 18.21 + 18.22 + // Mark the object if it's not already marked. This is used to mark 18.23 + // objects pointed to by roots that have been forwarded during a 18.24 + // GC. It is MT-safe. 18.25 + void mark_forwarded_object(oop from_obj, oop to_obj); 18.26 + 18.27 + oop copy_to_survivor_space(oop obj); 18.28 + 18.29 public: 18.30 G1ParCopyHelper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state, 18.31 G1ParScanClosure *scanner) : 18.32 G1ParClosureSuper(g1, par_scan_state), _scanner(scanner) { } 18.33 }; 18.34 18.35 -template<bool do_gen_barrier, G1Barrier barrier, 18.36 - bool do_mark_object> 18.37 +template <bool do_gen_barrier, G1Barrier barrier, bool do_mark_object> 18.38 class G1ParCopyClosure : public G1ParCopyHelper { 18.39 G1ParScanClosure _scanner; 18.40 18.41 @@ -140,9 +148,8 @@ 18.42 public: 18.43 G1ParCopyClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state, 18.44 ReferenceProcessor* rp) : 18.45 - _scanner(g1, par_scan_state, rp), 18.46 - G1ParCopyHelper(g1, par_scan_state, &_scanner) 18.47 - { 18.48 + _scanner(g1, par_scan_state, rp), 18.49 + G1ParCopyHelper(g1, par_scan_state, &_scanner) { 18.50 assert(_ref_processor == NULL, "sanity"); 18.51 } 18.52
19.1 --- a/src/share/vm/gc_implementation/g1/g1_globals.hpp Tue Jan 17 13:08:52 2012 -0500 19.2 +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp Tue Jan 17 21:25:28 2012 -0500 19.3 @@ -295,7 +295,7 @@ 19.4 "Percentage (0-100) of the heap size to use as minimum " \ 19.5 "young gen size.") \ 19.6 \ 19.7 - develop(uintx, G1DefaultMaxNewGenPercent, 50, \ 19.8 + develop(uintx, G1DefaultMaxNewGenPercent, 80, \ 19.9 "Percentage (0-100) of the heap size to use as maximum " \ 19.10 "young gen size.") 19.11
20.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.cpp Tue Jan 17 13:08:52 2012 -0500 20.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp Tue Jan 17 21:25:28 2012 -0500 20.3 @@ -1,5 +1,5 @@ 20.4 /* 20.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 20.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 20.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 20.8 * 20.9 * This code is free software; you can redistribute it and/or modify it 20.10 @@ -575,6 +575,40 @@ 20.11 oops_in_mr_iterate(MemRegion(bottom(), saved_mark_word()), cl); 20.12 } 20.13 20.14 +void HeapRegion::note_self_forwarding_removal_start(bool during_initial_mark, 20.15 + bool during_conc_mark) { 20.16 + // We always recreate the prev marking info and we'll explicitly 20.17 + // mark all objects we find to be self-forwarded on the prev 20.18 + // bitmap. So all objects need to be below PTAMS. 20.19 + _prev_top_at_mark_start = top(); 20.20 + _prev_marked_bytes = 0; 20.21 + 20.22 + if (during_initial_mark) { 20.23 + // During initial-mark, we'll also explicitly mark all objects 20.24 + // we find to be self-forwarded on the next bitmap. So all 20.25 + // objects need to be below NTAMS. 20.26 + _next_top_at_mark_start = top(); 20.27 + set_top_at_conc_mark_count(bottom()); 20.28 + _next_marked_bytes = 0; 20.29 + } else if (during_conc_mark) { 20.30 + // During concurrent mark, all objects in the CSet (including 20.31 + // the ones we find to be self-forwarded) are implicitly live. 20.32 + // So all objects need to be above NTAMS. 20.33 + _next_top_at_mark_start = bottom(); 20.34 + set_top_at_conc_mark_count(bottom()); 20.35 + _next_marked_bytes = 0; 20.36 + } 20.37 +} 20.38 + 20.39 +void HeapRegion::note_self_forwarding_removal_end(bool during_initial_mark, 20.40 + bool during_conc_mark, 20.41 + size_t marked_bytes) { 20.42 + assert(0 <= marked_bytes && marked_bytes <= used(), 20.43 + err_msg("marked: "SIZE_FORMAT" used: "SIZE_FORMAT, 20.44 + marked_bytes, used())); 20.45 + _prev_marked_bytes = marked_bytes; 20.46 +} 20.47 + 20.48 HeapWord* 20.49 HeapRegion::object_iterate_mem_careful(MemRegion mr, 20.50 ObjectClosure* cl) {
21.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.hpp Tue Jan 17 13:08:52 2012 -0500 21.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp Tue Jan 17 21:25:28 2012 -0500 21.3 @@ -1,5 +1,5 @@ 21.4 /* 21.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 21.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 21.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 21.8 * 21.9 * This code is free software; you can redistribute it and/or modify it 21.10 @@ -373,7 +373,8 @@ 21.11 ScrubRemSetClaimValue = 3, 21.12 ParVerifyClaimValue = 4, 21.13 RebuildRSClaimValue = 5, 21.14 - CompleteMarkCSetClaimValue = 6 21.15 + CompleteMarkCSetClaimValue = 6, 21.16 + ParEvacFailureClaimValue = 7 21.17 }; 21.18 21.19 inline HeapWord* par_allocate_no_bot_updates(size_t word_size) { 21.20 @@ -582,37 +583,33 @@ 21.21 // that the collector is about to start or has finished (concurrently) 21.22 // marking the heap. 21.23 21.24 - // Note the start of a marking phase. Record the 21.25 - // start of the unmarked area of the region here. 21.26 - void note_start_of_marking(bool during_initial_mark) { 21.27 - init_top_at_conc_mark_count(); 21.28 - _next_marked_bytes = 0; 21.29 - if (during_initial_mark && is_young() && !is_survivor()) 21.30 - _next_top_at_mark_start = bottom(); 21.31 - else 21.32 - _next_top_at_mark_start = top(); 21.33 - } 21.34 + // Notify the region that concurrent marking is starting. Initialize 21.35 + // all fields related to the next marking info. 21.36 + inline void note_start_of_marking(); 21.37 21.38 - // Note the end of a marking phase. Install the start of 21.39 - // the unmarked area that was captured at start of marking. 21.40 - void note_end_of_marking() { 21.41 - _prev_top_at_mark_start = _next_top_at_mark_start; 21.42 - _prev_marked_bytes = _next_marked_bytes; 21.43 - _next_marked_bytes = 0; 21.44 + // Notify the region that concurrent marking has finished. Copy the 21.45 + // (now finalized) next marking info fields into the prev marking 21.46 + // info fields. 21.47 + inline void note_end_of_marking(); 21.48 21.49 - guarantee(_prev_marked_bytes <= 21.50 - (size_t) (prev_top_at_mark_start() - bottom()) * HeapWordSize, 21.51 - "invariant"); 21.52 - } 21.53 + // Notify the region that it will be used as to-space during a GC 21.54 + // and we are about to start copying objects into it. 21.55 + inline void note_start_of_copying(bool during_initial_mark); 21.56 21.57 - // After an evacuation, we need to update _next_top_at_mark_start 21.58 - // to be the current top. Note this is only valid if we have only 21.59 - // ever evacuated into this region. If we evacuate, allocate, and 21.60 - // then evacuate we are in deep doodoo. 21.61 - void note_end_of_copying() { 21.62 - assert(top() >= _next_top_at_mark_start, "Increase only"); 21.63 - _next_top_at_mark_start = top(); 21.64 - } 21.65 + // Notify the region that it ceases being to-space during a GC and 21.66 + // we will not copy objects into it any more. 21.67 + inline void note_end_of_copying(bool during_initial_mark); 21.68 + 21.69 + // Notify the region that we are about to start processing 21.70 + // self-forwarded objects during evac failure handling. 21.71 + void note_self_forwarding_removal_start(bool during_initial_mark, 21.72 + bool during_conc_mark); 21.73 + 21.74 + // Notify the region that we have finished processing self-forwarded 21.75 + // objects during evac failure handling. 21.76 + void note_self_forwarding_removal_end(bool during_initial_mark, 21.77 + bool during_conc_mark, 21.78 + size_t marked_bytes); 21.79 21.80 // Returns "false" iff no object in the region was allocated when the 21.81 // last mark phase ended.
22.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.inline.hpp Tue Jan 17 13:08:52 2012 -0500 22.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.inline.hpp Tue Jan 17 21:25:28 2012 -0500 22.3 @@ -1,5 +1,5 @@ 22.4 /* 22.5 - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 22.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 22.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 22.8 * 22.9 * This code is free software; you can redistribute it and/or modify it 22.10 @@ -55,4 +55,71 @@ 22.11 return _offsets.block_start_const(p); 22.12 } 22.13 22.14 +inline void HeapRegion::note_start_of_marking() { 22.15 + init_top_at_conc_mark_count(); 22.16 + _next_marked_bytes = 0; 22.17 + _next_top_at_mark_start = top(); 22.18 +} 22.19 + 22.20 +inline void HeapRegion::note_end_of_marking() { 22.21 + _prev_top_at_mark_start = _next_top_at_mark_start; 22.22 + _prev_marked_bytes = _next_marked_bytes; 22.23 + _next_marked_bytes = 0; 22.24 + 22.25 + assert(_prev_marked_bytes <= 22.26 + (size_t) pointer_delta(prev_top_at_mark_start(), bottom()) * 22.27 + HeapWordSize, "invariant"); 22.28 +} 22.29 + 22.30 +inline void HeapRegion::note_start_of_copying(bool during_initial_mark) { 22.31 + if (during_initial_mark) { 22.32 + if (is_survivor()) { 22.33 + assert(false, "should not allocate survivors during IM"); 22.34 + } else { 22.35 + // During initial-mark we'll explicitly mark any objects on old 22.36 + // regions that are pointed to by roots. Given that explicit 22.37 + // marks only make sense under NTAMS it'd be nice if we could 22.38 + // check that condition if we wanted to. Given that we don't 22.39 + // know where the top of this region will end up, we simply set 22.40 + // NTAMS to the end of the region so all marks will be below 22.41 + // NTAMS. We'll set it to the actual top when we retire this region. 22.42 + _next_top_at_mark_start = end(); 22.43 + } 22.44 + } else { 22.45 + if (is_survivor()) { 22.46 + // This is how we always allocate survivors. 22.47 + assert(_next_top_at_mark_start == bottom(), "invariant"); 22.48 + } else { 22.49 + // We could have re-used this old region as to-space over a 22.50 + // couple of GCs since the start of the concurrent marking 22.51 + // cycle. This means that [bottom,NTAMS) will contain objects 22.52 + // copied up to and including initial-mark and [NTAMS, top) 22.53 + // will contain objects copied during the concurrent marking cycle. 22.54 + assert(top() >= _next_top_at_mark_start, "invariant"); 22.55 + } 22.56 + } 22.57 +} 22.58 + 22.59 +inline void HeapRegion::note_end_of_copying(bool during_initial_mark) { 22.60 + if (during_initial_mark) { 22.61 + if (is_survivor()) { 22.62 + assert(false, "should not allocate survivors during IM"); 22.63 + } else { 22.64 + // See the comment for note_start_of_copying() for the details 22.65 + // on this. 22.66 + assert(_next_top_at_mark_start == end(), "pre-condition"); 22.67 + _next_top_at_mark_start = top(); 22.68 + } 22.69 + } else { 22.70 + if (is_survivor()) { 22.71 + // This is how we always allocate survivors. 22.72 + assert(_next_top_at_mark_start == bottom(), "invariant"); 22.73 + } else { 22.74 + // See the comment for note_start_of_copying() for the details 22.75 + // on this. 22.76 + assert(top() >= _next_top_at_mark_start, "invariant"); 22.77 + } 22.78 + } 22.79 +} 22.80 + 22.81 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGION_INLINE_HPP
23.1 --- a/src/share/vm/gc_implementation/g1/ptrQueue.hpp Tue Jan 17 13:08:52 2012 -0500 23.2 +++ b/src/share/vm/gc_implementation/g1/ptrQueue.hpp Tue Jan 17 21:25:28 2012 -0500 23.3 @@ -1,5 +1,5 @@ 23.4 /* 23.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 23.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 23.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 23.8 * 23.9 * This code is free software; you can redistribute it and/or modify it 23.10 @@ -70,7 +70,7 @@ 23.11 // given PtrQueueSet. 23.12 PtrQueue(PtrQueueSet* qset, bool perm = false, bool active = false); 23.13 // Release any contained resources. 23.14 - void flush(); 23.15 + virtual void flush(); 23.16 // Calls flush() when destroyed. 23.17 ~PtrQueue() { flush(); } 23.18
24.1 --- a/src/share/vm/gc_implementation/g1/satbQueue.cpp Tue Jan 17 13:08:52 2012 -0500 24.2 +++ b/src/share/vm/gc_implementation/g1/satbQueue.cpp Tue Jan 17 21:25:28 2012 -0500 24.3 @@ -1,5 +1,5 @@ 24.4 /* 24.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 24.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 24.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 24.8 * 24.9 * This code is free software; you can redistribute it and/or modify it 24.10 @@ -31,6 +31,14 @@ 24.11 #include "runtime/thread.hpp" 24.12 #include "runtime/vmThread.hpp" 24.13 24.14 +void ObjPtrQueue::flush() { 24.15 + // The buffer might contain refs into the CSet. We have to filter it 24.16 + // first before we flush it, otherwise we might end up with an 24.17 + // enqueued buffer with refs into the CSet which breaks our invariants. 24.18 + filter(); 24.19 + PtrQueue::flush(); 24.20 +} 24.21 + 24.22 // This method removes entries from an SATB buffer that will not be 24.23 // useful to the concurrent marking threads. An entry is removed if it 24.24 // satisfies one of the following conditions: 24.25 @@ -44,38 +52,27 @@ 24.26 // process it again). 24.27 // 24.28 // The rest of the entries will be retained and are compacted towards 24.29 -// the top of the buffer. If with this filtering we clear a large 24.30 -// enough chunk of the buffer we can re-use it (instead of enqueueing 24.31 -// it) and we can just allow the mutator to carry on executing. 24.32 +// the top of the buffer. Note that, because we do not allow old 24.33 +// regions in the CSet during marking, all objects on the CSet regions 24.34 +// are young (eden or survivors) and therefore implicitly live. So any 24.35 +// references into the CSet will be removed during filtering. 24.36 24.37 -bool ObjPtrQueue::should_enqueue_buffer() { 24.38 - assert(_lock == NULL || _lock->owned_by_self(), 24.39 - "we should have taken the lock before calling this"); 24.40 - 24.41 - // A value of 0 means "don't filter SATB buffers". 24.42 - if (G1SATBBufferEnqueueingThresholdPercent == 0) { 24.43 - return true; 24.44 - } 24.45 - 24.46 +void ObjPtrQueue::filter() { 24.47 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 24.48 - 24.49 - // This method should only be called if there is a non-NULL buffer 24.50 - // that is full. 24.51 - assert(_index == 0, "pre-condition"); 24.52 - assert(_buf != NULL, "pre-condition"); 24.53 - 24.54 void** buf = _buf; 24.55 size_t sz = _sz; 24.56 24.57 + if (buf == NULL) { 24.58 + // nothing to do 24.59 + return; 24.60 + } 24.61 + 24.62 // Used for sanity checking at the end of the loop. 24.63 debug_only(size_t entries = 0; size_t retained = 0;) 24.64 24.65 size_t i = sz; 24.66 size_t new_index = sz; 24.67 24.68 - // Given that we are expecting _index == 0, we could have changed 24.69 - // the loop condition to (i > 0). But we are using _index for 24.70 - // generality. 24.71 while (i > _index) { 24.72 assert(i > 0, "we should have at least one more entry to process"); 24.73 i -= oopSize; 24.74 @@ -103,22 +100,58 @@ 24.75 debug_only(retained += 1;) 24.76 } 24.77 } 24.78 + 24.79 +#ifdef ASSERT 24.80 size_t entries_calc = (sz - _index) / oopSize; 24.81 assert(entries == entries_calc, "the number of entries we counted " 24.82 "should match the number of entries we calculated"); 24.83 size_t retained_calc = (sz - new_index) / oopSize; 24.84 assert(retained == retained_calc, "the number of retained entries we counted " 24.85 "should match the number of retained entries we calculated"); 24.86 - size_t perc = retained_calc * 100 / entries_calc; 24.87 +#endif // ASSERT 24.88 + 24.89 + _index = new_index; 24.90 +} 24.91 + 24.92 +// This method will first apply the above filtering to the buffer. If 24.93 +// post-filtering a large enough chunk of the buffer has been cleared 24.94 +// we can re-use the buffer (instead of enqueueing it) and we can just 24.95 +// allow the mutator to carry on executing using the same buffer 24.96 +// instead of replacing it. 24.97 + 24.98 +bool ObjPtrQueue::should_enqueue_buffer() { 24.99 + assert(_lock == NULL || _lock->owned_by_self(), 24.100 + "we should have taken the lock before calling this"); 24.101 + 24.102 + // Even if G1SATBBufferEnqueueingThresholdPercent == 0 we have to 24.103 + // filter the buffer given that this will remove any references into 24.104 + // the CSet as we currently assume that no such refs will appear in 24.105 + // enqueued buffers. 24.106 + 24.107 + // This method should only be called if there is a non-NULL buffer 24.108 + // that is full. 24.109 + assert(_index == 0, "pre-condition"); 24.110 + assert(_buf != NULL, "pre-condition"); 24.111 + 24.112 + filter(); 24.113 + 24.114 + size_t sz = _sz; 24.115 + size_t all_entries = sz / oopSize; 24.116 + size_t retained_entries = (sz - _index) / oopSize; 24.117 + size_t perc = retained_entries * 100 / all_entries; 24.118 bool should_enqueue = perc > (size_t) G1SATBBufferEnqueueingThresholdPercent; 24.119 - _index = new_index; 24.120 - 24.121 return should_enqueue; 24.122 } 24.123 24.124 void ObjPtrQueue::apply_closure(ObjectClosure* cl) { 24.125 if (_buf != NULL) { 24.126 apply_closure_to_buffer(cl, _buf, _index, _sz); 24.127 + } 24.128 +} 24.129 + 24.130 +void ObjPtrQueue::apply_closure_and_empty(ObjectClosure* cl) { 24.131 + if (_buf != NULL) { 24.132 + apply_closure_to_buffer(cl, _buf, _index, _sz); 24.133 _index = _sz; 24.134 } 24.135 } 24.136 @@ -135,6 +168,21 @@ 24.137 } 24.138 } 24.139 24.140 +#ifndef PRODUCT 24.141 +// Helpful for debugging 24.142 + 24.143 +void ObjPtrQueue::print(const char* name) { 24.144 + print(name, _buf, _index, _sz); 24.145 +} 24.146 + 24.147 +void ObjPtrQueue::print(const char* name, 24.148 + void** buf, size_t index, size_t sz) { 24.149 + gclog_or_tty->print_cr(" SATB BUFFER [%s] buf: "PTR_FORMAT" " 24.150 + "index: "SIZE_FORMAT" sz: "SIZE_FORMAT, 24.151 + name, buf, index, sz); 24.152 +} 24.153 +#endif // PRODUCT 24.154 + 24.155 #ifdef ASSERT 24.156 void ObjPtrQueue::verify_oops_in_buffer() { 24.157 if (_buf == NULL) return; 24.158 @@ -150,12 +198,9 @@ 24.159 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list 24.160 #endif // _MSC_VER 24.161 24.162 - 24.163 SATBMarkQueueSet::SATBMarkQueueSet() : 24.164 - PtrQueueSet(), 24.165 - _closure(NULL), _par_closures(NULL), 24.166 - _shared_satb_queue(this, true /*perm*/) 24.167 -{} 24.168 + PtrQueueSet(), _closure(NULL), _par_closures(NULL), 24.169 + _shared_satb_queue(this, true /*perm*/) { } 24.170 24.171 void SATBMarkQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock, 24.172 int process_completed_threshold, 24.173 @@ -167,7 +212,6 @@ 24.174 } 24.175 } 24.176 24.177 - 24.178 void SATBMarkQueueSet::handle_zero_index_for_thread(JavaThread* t) { 24.179 DEBUG_ONLY(t->satb_mark_queue().verify_oops_in_buffer();) 24.180 t->satb_mark_queue().handle_zero_index(); 24.181 @@ -228,6 +272,13 @@ 24.182 } 24.183 } 24.184 24.185 +void SATBMarkQueueSet::filter_thread_buffers() { 24.186 + for(JavaThread* t = Threads::first(); t; t = t->next()) { 24.187 + t->satb_mark_queue().filter(); 24.188 + } 24.189 + shared_satb_queue()->filter(); 24.190 +} 24.191 + 24.192 void SATBMarkQueueSet::set_closure(ObjectClosure* closure) { 24.193 _closure = closure; 24.194 } 24.195 @@ -239,9 +290,9 @@ 24.196 24.197 void SATBMarkQueueSet::iterate_closure_all_threads() { 24.198 for(JavaThread* t = Threads::first(); t; t = t->next()) { 24.199 - t->satb_mark_queue().apply_closure(_closure); 24.200 + t->satb_mark_queue().apply_closure_and_empty(_closure); 24.201 } 24.202 - shared_satb_queue()->apply_closure(_closure); 24.203 + shared_satb_queue()->apply_closure_and_empty(_closure); 24.204 } 24.205 24.206 void SATBMarkQueueSet::par_iterate_closure_all_threads(int worker) { 24.207 @@ -250,7 +301,7 @@ 24.208 24.209 for(JavaThread* t = Threads::first(); t; t = t->next()) { 24.210 if (t->claim_oops_do(true, parity)) { 24.211 - t->satb_mark_queue().apply_closure(_par_closures[worker]); 24.212 + t->satb_mark_queue().apply_closure_and_empty(_par_closures[worker]); 24.213 } 24.214 } 24.215 24.216 @@ -264,7 +315,7 @@ 24.217 24.218 VMThread* vmt = VMThread::vm_thread(); 24.219 if (vmt->claim_oops_do(true, parity)) { 24.220 - shared_satb_queue()->apply_closure(_par_closures[worker]); 24.221 + shared_satb_queue()->apply_closure_and_empty(_par_closures[worker]); 24.222 } 24.223 } 24.224 24.225 @@ -292,6 +343,61 @@ 24.226 } 24.227 } 24.228 24.229 +void SATBMarkQueueSet::iterate_completed_buffers_read_only(ObjectClosure* cl) { 24.230 + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); 24.231 + assert(cl != NULL, "pre-condition"); 24.232 + 24.233 + BufferNode* nd = _completed_buffers_head; 24.234 + while (nd != NULL) { 24.235 + void** buf = BufferNode::make_buffer_from_node(nd); 24.236 + ObjPtrQueue::apply_closure_to_buffer(cl, buf, 0, _sz); 24.237 + nd = nd->next(); 24.238 + } 24.239 +} 24.240 + 24.241 +void SATBMarkQueueSet::iterate_thread_buffers_read_only(ObjectClosure* cl) { 24.242 + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); 24.243 + assert(cl != NULL, "pre-condition"); 24.244 + 24.245 + for (JavaThread* t = Threads::first(); t; t = t->next()) { 24.246 + t->satb_mark_queue().apply_closure(cl); 24.247 + } 24.248 + shared_satb_queue()->apply_closure(cl); 24.249 +} 24.250 + 24.251 +#ifndef PRODUCT 24.252 +// Helpful for debugging 24.253 + 24.254 +#define SATB_PRINTER_BUFFER_SIZE 256 24.255 + 24.256 +void SATBMarkQueueSet::print_all(const char* msg) { 24.257 + char buffer[SATB_PRINTER_BUFFER_SIZE]; 24.258 + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); 24.259 + 24.260 + gclog_or_tty->cr(); 24.261 + gclog_or_tty->print_cr("SATB BUFFERS [%s]", msg); 24.262 + 24.263 + BufferNode* nd = _completed_buffers_head; 24.264 + int i = 0; 24.265 + while (nd != NULL) { 24.266 + void** buf = BufferNode::make_buffer_from_node(nd); 24.267 + jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Enqueued: %d", i); 24.268 + ObjPtrQueue::print(buffer, buf, 0, _sz); 24.269 + nd = nd->next(); 24.270 + i += 1; 24.271 + } 24.272 + 24.273 + for (JavaThread* t = Threads::first(); t; t = t->next()) { 24.274 + jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Thread: %s", t->name()); 24.275 + t->satb_mark_queue().print(buffer); 24.276 + } 24.277 + 24.278 + shared_satb_queue()->print("Shared"); 24.279 + 24.280 + gclog_or_tty->cr(); 24.281 +} 24.282 +#endif // PRODUCT 24.283 + 24.284 void SATBMarkQueueSet::abandon_partial_marking() { 24.285 BufferNode* buffers_to_delete = NULL; 24.286 { 24.287 @@ -316,5 +422,5 @@ 24.288 for (JavaThread* t = Threads::first(); t; t = t->next()) { 24.289 t->satb_mark_queue().reset(); 24.290 } 24.291 - shared_satb_queue()->reset(); 24.292 + shared_satb_queue()->reset(); 24.293 }
25.1 --- a/src/share/vm/gc_implementation/g1/satbQueue.hpp Tue Jan 17 13:08:52 2012 -0500 25.2 +++ b/src/share/vm/gc_implementation/g1/satbQueue.hpp Tue Jan 17 21:25:28 2012 -0500 25.3 @@ -1,5 +1,5 @@ 25.4 /* 25.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. 25.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. 25.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 25.8 * 25.9 * This code is free software; you can redistribute it and/or modify it 25.10 @@ -29,9 +29,26 @@ 25.11 25.12 class ObjectClosure; 25.13 class JavaThread; 25.14 +class SATBMarkQueueSet; 25.15 25.16 // A ptrQueue whose elements are "oops", pointers to object heads. 25.17 class ObjPtrQueue: public PtrQueue { 25.18 + friend class SATBMarkQueueSet; 25.19 + 25.20 +private: 25.21 + // Filter out unwanted entries from the buffer. 25.22 + void filter(); 25.23 + 25.24 + // Apply the closure to all elements. 25.25 + void apply_closure(ObjectClosure* cl); 25.26 + 25.27 + // Apply the closure to all elements and empty the buffer; 25.28 + void apply_closure_and_empty(ObjectClosure* cl); 25.29 + 25.30 + // Apply the closure to all elements of "buf", down to "index" (inclusive.) 25.31 + static void apply_closure_to_buffer(ObjectClosure* cl, 25.32 + void** buf, size_t index, size_t sz); 25.33 + 25.34 public: 25.35 ObjPtrQueue(PtrQueueSet* qset, bool perm = false) : 25.36 // SATB queues are only active during marking cycles. We create 25.37 @@ -41,23 +58,23 @@ 25.38 // field to true. This is done in JavaThread::initialize_queues(). 25.39 PtrQueue(qset, perm, false /* active */) { } 25.40 25.41 + // Overrides PtrQueue::flush() so that it can filter the buffer 25.42 + // before it is flushed. 25.43 + virtual void flush(); 25.44 + 25.45 // Overrides PtrQueue::should_enqueue_buffer(). See the method's 25.46 // definition for more information. 25.47 virtual bool should_enqueue_buffer(); 25.48 25.49 - // Apply the closure to all elements, and reset the index to make the 25.50 - // buffer empty. 25.51 - void apply_closure(ObjectClosure* cl); 25.52 - 25.53 - // Apply the closure to all elements of "buf", down to "index" (inclusive.) 25.54 - static void apply_closure_to_buffer(ObjectClosure* cl, 25.55 - void** buf, size_t index, size_t sz); 25.56 +#ifndef PRODUCT 25.57 + // Helpful for debugging 25.58 + void print(const char* name); 25.59 + static void print(const char* name, void** buf, size_t index, size_t sz); 25.60 +#endif // PRODUCT 25.61 25.62 void verify_oops_in_buffer() NOT_DEBUG_RETURN; 25.63 }; 25.64 25.65 - 25.66 - 25.67 class SATBMarkQueueSet: public PtrQueueSet { 25.68 ObjectClosure* _closure; 25.69 ObjectClosure** _par_closures; // One per ParGCThread. 25.70 @@ -88,6 +105,9 @@ 25.71 // set itself, has an active value same as expected_active. 25.72 void set_active_all_threads(bool b, bool expected_active); 25.73 25.74 + // Filter all the currently-active SATB buffers. 25.75 + void filter_thread_buffers(); 25.76 + 25.77 // Register "blk" as "the closure" for all queues. Only one such closure 25.78 // is allowed. The "apply_closure_to_completed_buffer" method will apply 25.79 // this closure to a completed buffer, and "iterate_closure_all_threads" 25.80 @@ -98,10 +118,9 @@ 25.81 // closures, one for each parallel GC thread. 25.82 void set_par_closure(int i, ObjectClosure* closure); 25.83 25.84 - // If there is a registered closure for buffers, apply it to all entries 25.85 - // in all currently-active buffers. This should only be applied at a 25.86 - // safepoint. (Currently must not be called in parallel; this should 25.87 - // change in the future.) 25.88 + // Apply the registered closure to all entries on each 25.89 + // currently-active buffer and then empty the buffer. It should only 25.90 + // be called serially and at a safepoint. 25.91 void iterate_closure_all_threads(); 25.92 // Parallel version of the above. 25.93 void par_iterate_closure_all_threads(int worker); 25.94 @@ -117,11 +136,21 @@ 25.95 return apply_closure_to_completed_buffer_work(true, worker); 25.96 } 25.97 25.98 + // Apply the given closure on enqueued and currently-active buffers 25.99 + // respectively. Both methods are read-only, i.e., they do not 25.100 + // modify any of the buffers. 25.101 + void iterate_completed_buffers_read_only(ObjectClosure* cl); 25.102 + void iterate_thread_buffers_read_only(ObjectClosure* cl); 25.103 + 25.104 +#ifndef PRODUCT 25.105 + // Helpful for debugging 25.106 + void print_all(const char* msg); 25.107 +#endif // PRODUCT 25.108 + 25.109 ObjPtrQueue* shared_satb_queue() { return &_shared_satb_queue; } 25.110 25.111 // If a marking is being abandoned, reset any unprocessed log buffers. 25.112 void abandon_partial_marking(); 25.113 - 25.114 }; 25.115 25.116 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_SATBQUEUE_HPP
26.1 --- a/src/share/vm/oops/instanceKlass.hpp Tue Jan 17 13:08:52 2012 -0500 26.2 +++ b/src/share/vm/oops/instanceKlass.hpp Tue Jan 17 21:25:28 2012 -0500 26.3 @@ -231,6 +231,10 @@ 26.4 u2 _java_fields_count; // The number of declared Java fields 26.5 int _nonstatic_oop_map_size;// size in words of nonstatic oop map blocks 26.6 26.7 + bool _is_marked_dependent; // used for marking during flushing and deoptimization 26.8 + bool _rewritten; // methods rewritten. 26.9 + bool _has_nonstatic_fields; // for sizing with UseCompressedOops 26.10 + bool _should_verify_class; // allow caching of preverification 26.11 u2 _minor_version; // minor version number of class file 26.12 u2 _major_version; // major version number of class file 26.13 Thread* _init_thread; // Pointer to current thread doing initialization (to handle recusive initialization) 26.14 @@ -261,19 +265,6 @@ 26.15 // _idnum_allocated_count. 26.16 u1 _init_state; // state of class 26.17 26.18 - // Compact the following four boolean flags into 1-bit each. These four flags 26.19 - // were defined as separate boolean fields and each was 1-byte before. Since 26.20 - // there are 2 bytes unused after the _idnum_allocated_count field, place the 26.21 - // _misc_flags field after _idnum_allocated_count to utilize the unused bits 26.22 - // and save total 4-bytes. 26.23 - enum { 26.24 - IS_MARKED_DEPENDENT = 0x1, // used for marking during flushing and deoptimization 26.25 - REWRITTEN = 0x2, // methods rewritten. 26.26 - HAS_NONSTATIC_FIELDS = 0x4, // for sizing with UseCompressedOops 26.27 - SHOULD_VERIFY_CLASS = 0x8 // allow caching of preverification 26.28 - }; 26.29 - u1 _misc_flags; 26.30 - 26.31 // embedded Java vtable follows here 26.32 // embedded Java itables follows here 26.33 // embedded static fields follows here 26.34 @@ -283,14 +274,8 @@ 26.35 friend class SystemDictionary; 26.36 26.37 public: 26.38 - bool has_nonstatic_fields() const { return (_misc_flags & HAS_NONSTATIC_FIELDS) != 0; } 26.39 - void set_has_nonstatic_fields(bool b) { 26.40 - if (b) { 26.41 - _misc_flags |= HAS_NONSTATIC_FIELDS; 26.42 - } else { 26.43 - _misc_flags &= ~HAS_NONSTATIC_FIELDS; 26.44 - } 26.45 - } 26.46 + bool has_nonstatic_fields() const { return _has_nonstatic_fields; } 26.47 + void set_has_nonstatic_fields(bool b) { _has_nonstatic_fields = b; } 26.48 26.49 // field sizes 26.50 int nonstatic_field_size() const { return _nonstatic_field_size; } 26.51 @@ -398,23 +383,15 @@ 26.52 bool is_in_error_state() const { return _init_state == initialization_error; } 26.53 bool is_reentrant_initialization(Thread *thread) { return thread == _init_thread; } 26.54 ClassState init_state() { return (ClassState)_init_state; } 26.55 - bool is_rewritten() const { return (_misc_flags & REWRITTEN) != 0; } 26.56 + bool is_rewritten() const { return _rewritten; } 26.57 26.58 // defineClass specified verification 26.59 - bool should_verify_class() const { return (_misc_flags & SHOULD_VERIFY_CLASS) != 0; } 26.60 - void set_should_verify_class(bool value) { 26.61 - if (value) { 26.62 - _misc_flags |= SHOULD_VERIFY_CLASS; 26.63 - } else { 26.64 - _misc_flags &= ~SHOULD_VERIFY_CLASS; 26.65 - } 26.66 - } 26.67 - 26.68 + bool should_verify_class() const { return _should_verify_class; } 26.69 + void set_should_verify_class(bool value) { _should_verify_class = value; } 26.70 26.71 // marking 26.72 - bool is_marked_dependent() const { return (_misc_flags & IS_MARKED_DEPENDENT) != 0; } 26.73 - void set_is_marked_dependent() { _misc_flags |= IS_MARKED_DEPENDENT; } 26.74 - void clear_is_marked_dependent() { _misc_flags &= ~IS_MARKED_DEPENDENT; } 26.75 + bool is_marked_dependent() const { return _is_marked_dependent; } 26.76 + void set_is_marked_dependent(bool value) { _is_marked_dependent = value; } 26.77 26.78 // initialization (virtuals from Klass) 26.79 bool should_be_initialized() const; // means that initialize should be called 26.80 @@ -784,7 +761,7 @@ 26.81 #else 26.82 void set_init_state(ClassState state) { _init_state = (u1)state; } 26.83 #endif 26.84 - void set_rewritten() { _misc_flags |= REWRITTEN; } 26.85 + void set_rewritten() { _rewritten = true; } 26.86 void set_init_thread(Thread *thread) { _init_thread = thread; } 26.87 26.88 u2 idnum_allocated_count() const { return _idnum_allocated_count; }
27.1 --- a/src/share/vm/oops/instanceKlassKlass.cpp Tue Jan 17 13:08:52 2012 -0500 27.2 +++ b/src/share/vm/oops/instanceKlassKlass.cpp Tue Jan 17 21:25:28 2012 -0500 27.3 @@ -399,7 +399,7 @@ 27.4 ik->set_inner_classes(NULL); 27.5 ik->set_static_oop_field_count(0); 27.6 ik->set_nonstatic_field_size(0); 27.7 - ik->clear_is_marked_dependent(); 27.8 + ik->set_is_marked_dependent(false); 27.9 ik->set_init_state(instanceKlass::allocated); 27.10 ik->set_init_thread(NULL); 27.11 ik->set_reference_type(rt);
28.1 --- a/src/share/vm/opto/c2_globals.hpp Tue Jan 17 13:08:52 2012 -0500 28.2 +++ b/src/share/vm/opto/c2_globals.hpp Tue Jan 17 21:25:28 2012 -0500 28.3 @@ -426,6 +426,9 @@ 28.4 product(bool, EliminateLocks, true, \ 28.5 "Coarsen locks when possible") \ 28.6 \ 28.7 + product(bool, EliminateNestedLocks, true, \ 28.8 + "Eliminate nested locks of the same object when possible") \ 28.9 + \ 28.10 notproduct(bool, PrintLockStatistics, false, \ 28.11 "Print precise statistics on the dynamic lock usage") \ 28.12 \
29.1 --- a/src/share/vm/opto/callnode.cpp Tue Jan 17 13:08:52 2012 -0500 29.2 +++ b/src/share/vm/opto/callnode.cpp Tue Jan 17 21:25:28 2012 -0500 29.3 @@ -400,10 +400,10 @@ 29.4 Node *box = mcall->monitor_box(this, i); 29.5 Node *obj = mcall->monitor_obj(this, i); 29.6 if ( OptoReg::is_valid(regalloc->get_reg_first(box)) ) { 29.7 - while( !box->is_BoxLock() ) box = box->in(1); 29.8 + box = BoxLockNode::box_node(box); 29.9 format_helper( regalloc, st, box, "MON-BOX[", i, &scobjs ); 29.10 } else { 29.11 - OptoReg::Name box_reg = BoxLockNode::stack_slot(box); 29.12 + OptoReg::Name box_reg = BoxLockNode::reg(box); 29.13 st->print(" MON-BOX%d=%s+%d", 29.14 i, 29.15 OptoReg::regname(OptoReg::c_frame_pointer), 29.16 @@ -411,8 +411,7 @@ 29.17 } 29.18 const char* obj_msg = "MON-OBJ["; 29.19 if (EliminateLocks) { 29.20 - while( !box->is_BoxLock() ) box = box->in(1); 29.21 - if (box->as_BoxLock()->is_eliminated()) 29.22 + if (BoxLockNode::box_node(box)->is_eliminated()) 29.23 obj_msg = "MON-OBJ(LOCK ELIMINATED)["; 29.24 } 29.25 format_helper( regalloc, st, obj, obj_msg, i, &scobjs ); 29.26 @@ -1387,8 +1386,9 @@ 29.27 Node *n = ctrl_proj->in(0); 29.28 if (n != NULL && n->is_Unlock()) { 29.29 UnlockNode *unlock = n->as_Unlock(); 29.30 - if ((lock->obj_node() == unlock->obj_node()) && 29.31 - (lock->box_node() == unlock->box_node()) && !unlock->is_eliminated()) { 29.32 + if (lock->obj_node()->eqv_uncast(unlock->obj_node()) && 29.33 + BoxLockNode::same_slot(lock->box_node(), unlock->box_node()) && 29.34 + !unlock->is_eliminated()) { 29.35 lock_ops.append(unlock); 29.36 return true; 29.37 } 29.38 @@ -1431,8 +1431,8 @@ 29.39 } 29.40 if (ctrl->is_Lock()) { 29.41 LockNode *lock = ctrl->as_Lock(); 29.42 - if ((lock->obj_node() == unlock->obj_node()) && 29.43 - (lock->box_node() == unlock->box_node())) { 29.44 + if (lock->obj_node()->eqv_uncast(unlock->obj_node()) && 29.45 + BoxLockNode::same_slot(lock->box_node(), unlock->box_node())) { 29.46 lock_result = lock; 29.47 } 29.48 } 29.49 @@ -1462,8 +1462,9 @@ 29.50 } 29.51 if (lock1_node != NULL && lock1_node->is_Lock()) { 29.52 LockNode *lock1 = lock1_node->as_Lock(); 29.53 - if ((lock->obj_node() == lock1->obj_node()) && 29.54 - (lock->box_node() == lock1->box_node()) && !lock1->is_eliminated()) { 29.55 + if (lock->obj_node()->eqv_uncast(lock1->obj_node()) && 29.56 + BoxLockNode::same_slot(lock->box_node(), lock1->box_node()) && 29.57 + !lock1->is_eliminated()) { 29.58 lock_ops.append(lock1); 29.59 return true; 29.60 } 29.61 @@ -1507,19 +1508,16 @@ 29.62 void AbstractLockNode::create_lock_counter(JVMState* state) { 29.63 _counter = OptoRuntime::new_named_counter(state, NamedCounter::LockCounter); 29.64 } 29.65 -#endif 29.66 29.67 -void AbstractLockNode::set_eliminated() { 29.68 - _eliminate = true; 29.69 -#ifndef PRODUCT 29.70 +void AbstractLockNode::set_eliminated_lock_counter() { 29.71 if (_counter) { 29.72 // Update the counter to indicate that this lock was eliminated. 29.73 // The counter update code will stay around even though the 29.74 // optimizer will eliminate the lock operation itself. 29.75 _counter->set_tag(NamedCounter::EliminatedLockCounter); 29.76 } 29.77 +} 29.78 #endif 29.79 -} 29.80 29.81 //============================================================================= 29.82 Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) { 29.83 @@ -1535,7 +1533,7 @@ 29.84 // prevents macro expansion from expanding the lock. Since we don't 29.85 // modify the graph, the value returned from this function is the 29.86 // one computed above. 29.87 - if (can_reshape && EliminateLocks && (!is_eliminated() || is_coarsened())) { 29.88 + if (can_reshape && EliminateLocks && !is_non_esc_obj()) { 29.89 // 29.90 // If we are locking an unescaped object, the lock/unlock is unnecessary 29.91 // 29.92 @@ -1544,16 +1542,11 @@ 29.93 if (cgr != NULL) 29.94 es = cgr->escape_state(obj_node()); 29.95 if (es != PointsToNode::UnknownEscape && es != PointsToNode::GlobalEscape) { 29.96 - if (!is_eliminated()) { 29.97 - // Mark it eliminated to update any counters 29.98 - this->set_eliminated(); 29.99 - } else { 29.100 - assert(is_coarsened(), "sanity"); 29.101 - // The lock could be marked eliminated by lock coarsening 29.102 - // code during first IGVN before EA. Clear coarsened flag 29.103 - // to eliminate all associated locks/unlocks. 29.104 - this->clear_coarsened(); 29.105 - } 29.106 + assert(!is_eliminated() || is_coarsened(), "sanity"); 29.107 + // The lock could be marked eliminated by lock coarsening 29.108 + // code during first IGVN before EA. Replace coarsened flag 29.109 + // to eliminate all associated locks/unlocks. 29.110 + this->set_non_esc_obj(); 29.111 return result; 29.112 } 29.113 29.114 @@ -1613,8 +1606,7 @@ 29.115 for (int i = 0; i < lock_ops.length(); i++) { 29.116 AbstractLockNode* lock = lock_ops.at(i); 29.117 29.118 - // Mark it eliminated to update any counters 29.119 - lock->set_eliminated(); 29.120 + // Mark it eliminated by coarsening and update any counters 29.121 lock->set_coarsened(); 29.122 } 29.123 } else if (ctrl->is_Region() && 29.124 @@ -1632,6 +1624,40 @@ 29.125 } 29.126 29.127 //============================================================================= 29.128 +bool LockNode::is_nested_lock_region() { 29.129 + BoxLockNode* box = box_node()->as_BoxLock(); 29.130 + int stk_slot = box->stack_slot(); 29.131 + if (stk_slot <= 0) 29.132 + return false; // External lock or it is not Box (Phi node). 29.133 + 29.134 + // Ignore complex cases: merged locks or multiple locks. 29.135 + Node* obj = obj_node(); 29.136 + LockNode* unique_lock = NULL; 29.137 + if (!box->is_simple_lock_region(&unique_lock, obj) || 29.138 + (unique_lock != this)) { 29.139 + return false; 29.140 + } 29.141 + 29.142 + // Look for external lock for the same object. 29.143 + SafePointNode* sfn = this->as_SafePoint(); 29.144 + JVMState* youngest_jvms = sfn->jvms(); 29.145 + int max_depth = youngest_jvms->depth(); 29.146 + for (int depth = 1; depth <= max_depth; depth++) { 29.147 + JVMState* jvms = youngest_jvms->of_depth(depth); 29.148 + int num_mon = jvms->nof_monitors(); 29.149 + // Loop over monitors 29.150 + for (int idx = 0; idx < num_mon; idx++) { 29.151 + Node* obj_node = sfn->monitor_obj(jvms, idx); 29.152 + BoxLockNode* box_node = sfn->monitor_box(jvms, idx)->as_BoxLock(); 29.153 + if ((box_node->stack_slot() < stk_slot) && obj_node->eqv_uncast(obj)) { 29.154 + return true; 29.155 + } 29.156 + } 29.157 + } 29.158 + return false; 29.159 +} 29.160 + 29.161 +//============================================================================= 29.162 uint UnlockNode::size_of() const { return sizeof(*this); } 29.163 29.164 //============================================================================= 29.165 @@ -1649,7 +1675,7 @@ 29.166 // modify the graph, the value returned from this function is the 29.167 // one computed above. 29.168 // Escape state is defined after Parse phase. 29.169 - if (can_reshape && EliminateLocks && (!is_eliminated() || is_coarsened())) { 29.170 + if (can_reshape && EliminateLocks && !is_non_esc_obj()) { 29.171 // 29.172 // If we are unlocking an unescaped object, the lock/unlock is unnecessary. 29.173 // 29.174 @@ -1658,16 +1684,11 @@ 29.175 if (cgr != NULL) 29.176 es = cgr->escape_state(obj_node()); 29.177 if (es != PointsToNode::UnknownEscape && es != PointsToNode::GlobalEscape) { 29.178 - if (!is_eliminated()) { 29.179 - // Mark it eliminated to update any counters 29.180 - this->set_eliminated(); 29.181 - } else { 29.182 - assert(is_coarsened(), "sanity"); 29.183 - // The lock could be marked eliminated by lock coarsening 29.184 - // code during first IGVN before EA. Clear coarsened flag 29.185 - // to eliminate all associated locks/unlocks. 29.186 - this->clear_coarsened(); 29.187 - } 29.188 + assert(!is_eliminated() || is_coarsened(), "sanity"); 29.189 + // The lock could be marked eliminated by lock coarsening 29.190 + // code during first IGVN before EA. Replace coarsened flag 29.191 + // to eliminate all associated locks/unlocks. 29.192 + this->set_non_esc_obj(); 29.193 } 29.194 } 29.195 return result;
30.1 --- a/src/share/vm/opto/callnode.hpp Tue Jan 17 13:08:52 2012 -0500 30.2 +++ b/src/share/vm/opto/callnode.hpp Tue Jan 17 21:25:28 2012 -0500 30.3 @@ -840,8 +840,12 @@ 30.4 //------------------------------AbstractLockNode----------------------------------- 30.5 class AbstractLockNode: public CallNode { 30.6 private: 30.7 - bool _eliminate; // indicates this lock can be safely eliminated 30.8 - bool _coarsened; // indicates this lock was coarsened 30.9 + enum { 30.10 + Regular = 0, // Normal lock 30.11 + NonEscObj, // Lock is used for non escaping object 30.12 + Coarsened, // Lock was coarsened 30.13 + Nested // Nested lock 30.14 + } _kind; 30.15 #ifndef PRODUCT 30.16 NamedCounter* _counter; 30.17 #endif 30.18 @@ -858,12 +862,13 @@ 30.19 GrowableArray<AbstractLockNode*> &lock_ops); 30.20 LockNode *find_matching_lock(UnlockNode* unlock); 30.21 30.22 + // Update the counter to indicate that this lock was eliminated. 30.23 + void set_eliminated_lock_counter() PRODUCT_RETURN; 30.24 30.25 public: 30.26 AbstractLockNode(const TypeFunc *tf) 30.27 : CallNode(tf, NULL, TypeRawPtr::BOTTOM), 30.28 - _coarsened(false), 30.29 - _eliminate(false) 30.30 + _kind(Regular) 30.31 { 30.32 #ifndef PRODUCT 30.33 _counter = NULL; 30.34 @@ -873,20 +878,23 @@ 30.35 Node * obj_node() const {return in(TypeFunc::Parms + 0); } 30.36 Node * box_node() const {return in(TypeFunc::Parms + 1); } 30.37 Node * fastlock_node() const {return in(TypeFunc::Parms + 2); } 30.38 + void set_box_node(Node* box) { set_req(TypeFunc::Parms + 1, box); } 30.39 + 30.40 const Type *sub(const Type *t1, const Type *t2) const { return TypeInt::CC;} 30.41 30.42 virtual uint size_of() const { return sizeof(*this); } 30.43 30.44 - bool is_eliminated() {return _eliminate; } 30.45 - // mark node as eliminated and update the counter if there is one 30.46 - void set_eliminated(); 30.47 + bool is_eliminated() const { return (_kind != Regular); } 30.48 + bool is_non_esc_obj() const { return (_kind == NonEscObj); } 30.49 + bool is_coarsened() const { return (_kind == Coarsened); } 30.50 + bool is_nested() const { return (_kind == Nested); } 30.51 30.52 - bool is_coarsened() { return _coarsened; } 30.53 - void set_coarsened() { _coarsened = true; } 30.54 - void clear_coarsened() { _coarsened = false; } 30.55 + void set_non_esc_obj() { _kind = NonEscObj; set_eliminated_lock_counter(); } 30.56 + void set_coarsened() { _kind = Coarsened; set_eliminated_lock_counter(); } 30.57 + void set_nested() { _kind = Nested; set_eliminated_lock_counter(); } 30.58 30.59 // locking does not modify its arguments 30.60 - virtual bool may_modify(const TypePtr *addr_t, PhaseTransform *phase){ return false;} 30.61 + virtual bool may_modify(const TypePtr *addr_t, PhaseTransform *phase){ return false;} 30.62 30.63 #ifndef PRODUCT 30.64 void create_lock_counter(JVMState* s); 30.65 @@ -936,6 +944,8 @@ 30.66 virtual void clone_jvms() { 30.67 set_jvms(jvms()->clone_deep(Compile::current())); 30.68 } 30.69 + 30.70 + bool is_nested_lock_region(); // Is this Lock nested? 30.71 }; 30.72 30.73 //------------------------------Unlock---------------------------------------
31.1 --- a/src/share/vm/opto/cfgnode.cpp Tue Jan 17 13:08:52 2012 -0500 31.2 +++ b/src/share/vm/opto/cfgnode.cpp Tue Jan 17 21:25:28 2012 -0500 31.3 @@ -1597,7 +1597,7 @@ 31.4 bool is_loop = (r->is_Loop() && r->req() == 3); 31.5 // Then, check if there is a data loop when phi references itself directly 31.6 // or through other data nodes. 31.7 - if (is_loop && !phase->eqv_uncast(uin, in(LoopNode::EntryControl)) || 31.8 + if (is_loop && !uin->eqv_uncast(in(LoopNode::EntryControl)) || 31.9 !is_loop && is_unsafe_data_reference(uin)) { 31.10 // Break this data loop to avoid creation of a dead loop. 31.11 if (can_reshape) {
32.1 --- a/src/share/vm/opto/chaitin.hpp Tue Jan 17 13:08:52 2012 -0500 32.2 +++ b/src/share/vm/opto/chaitin.hpp Tue Jan 17 21:25:28 2012 -0500 32.3 @@ -485,7 +485,11 @@ 32.4 return yank_if_dead(old, current_block, &value, ®nd); 32.5 } 32.6 32.7 - int yank_if_dead( Node *old, Block *current_block, Node_List *value, Node_List *regnd ); 32.8 + int yank_if_dead( Node *old, Block *current_block, Node_List *value, Node_List *regnd ) { 32.9 + return yank_if_dead_recurse(old, old, current_block, value, regnd); 32.10 + } 32.11 + int yank_if_dead_recurse(Node *old, Node *orig_old, Block *current_block, 32.12 + Node_List *value, Node_List *regnd); 32.13 int yank( Node *old, Block *current_block, Node_List *value, Node_List *regnd ); 32.14 int elide_copy( Node *n, int k, Block *current_block, Node_List &value, Node_List ®nd, bool can_change_regs ); 32.15 int use_prior_register( Node *copy, uint idx, Node *def, Block *current_block, Node_List &value, Node_List ®nd );
33.1 --- a/src/share/vm/opto/escape.cpp Tue Jan 17 13:08:52 2012 -0500 33.2 +++ b/src/share/vm/opto/escape.cpp Tue Jan 17 21:25:28 2012 -0500 33.3 @@ -1842,20 +1842,15 @@ 33.4 Node *n = C->macro_node(i); 33.5 if (n->is_AbstractLock()) { // Lock and Unlock nodes 33.6 AbstractLockNode* alock = n->as_AbstractLock(); 33.7 - if (!alock->is_eliminated() || alock->is_coarsened()) { 33.8 + if (!alock->is_non_esc_obj()) { 33.9 PointsToNode::EscapeState es = escape_state(alock->obj_node()); 33.10 assert(es != PointsToNode::UnknownEscape, "should know"); 33.11 if (es != PointsToNode::UnknownEscape && es != PointsToNode::GlobalEscape) { 33.12 - if (!alock->is_eliminated()) { 33.13 - // Mark it eliminated to update any counters 33.14 - alock->set_eliminated(); 33.15 - } else { 33.16 - // The lock could be marked eliminated by lock coarsening 33.17 - // code during first IGVN before EA. Clear coarsened flag 33.18 - // to eliminate all associated locks/unlocks and relock 33.19 - // during deoptimization. 33.20 - alock->clear_coarsened(); 33.21 - } 33.22 + assert(!alock->is_eliminated() || alock->is_coarsened(), "sanity"); 33.23 + // The lock could be marked eliminated by lock coarsening 33.24 + // code during first IGVN before EA. Replace coarsened flag 33.25 + // to eliminate all associated locks/unlocks. 33.26 + alock->set_non_esc_obj(); 33.27 } 33.28 } 33.29 }
34.1 --- a/src/share/vm/opto/library_call.cpp Tue Jan 17 13:08:52 2012 -0500 34.2 +++ b/src/share/vm/opto/library_call.cpp Tue Jan 17 21:25:28 2012 -0500 34.3 @@ -819,7 +819,7 @@ 34.4 if (stopped()) 34.5 return NULL; // already stopped 34.6 bool zero_offset = _gvn.type(offset) == TypeInt::ZERO; 34.7 - if (zero_offset && _gvn.eqv_uncast(subseq_length, array_length)) 34.8 + if (zero_offset && subseq_length->eqv_uncast(array_length)) 34.9 return NULL; // common case of whole-array copy 34.10 Node* last = subseq_length; 34.11 if (!zero_offset) // last += offset 34.12 @@ -4667,7 +4667,7 @@ 34.13 if (ReduceBulkZeroing 34.14 && !ZeroTLAB // pointless if already zeroed 34.15 && basic_elem_type != T_CONFLICT // avoid corner case 34.16 - && !_gvn.eqv_uncast(src, dest) 34.17 + && !src->eqv_uncast(dest) 34.18 && ((alloc = tightly_coupled_allocation(dest, slow_region)) 34.19 != NULL) 34.20 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0 34.21 @@ -4745,7 +4745,7 @@ 34.22 // copy_length is 0. 34.23 if (!stopped() && dest_uninitialized) { 34.24 Node* dest_length = alloc->in(AllocateNode::ALength); 34.25 - if (_gvn.eqv_uncast(copy_length, dest_length) 34.26 + if (copy_length->eqv_uncast(dest_length) 34.27 || _gvn.find_int_con(dest_length, 1) <= 0) { 34.28 // There is no zeroing to do. No need for a secondary raw memory barrier. 34.29 } else { 34.30 @@ -4791,7 +4791,7 @@ 34.31 // with its attendant messy index arithmetic, and upgrade 34.32 // the copy to a more hardware-friendly word size of 64 bits. 34.33 Node* tail_ctl = NULL; 34.34 - if (!stopped() && !_gvn.eqv_uncast(dest_tail, dest_length)) { 34.35 + if (!stopped() && !dest_tail->eqv_uncast(dest_length)) { 34.36 Node* cmp_lt = _gvn.transform( new(C,3) CmpINode(dest_tail, dest_length) ); 34.37 Node* bol_lt = _gvn.transform( new(C,2) BoolNode(cmp_lt, BoolTest::lt) ); 34.38 tail_ctl = generate_slow_guard(bol_lt, NULL);
35.1 --- a/src/share/vm/opto/locknode.cpp Tue Jan 17 13:08:52 2012 -0500 35.2 +++ b/src/share/vm/opto/locknode.cpp Tue Jan 17 21:25:28 2012 -0500 35.3 @@ -49,18 +49,22 @@ 35.4 35.5 //-----------------------------hash-------------------------------------------- 35.6 uint BoxLockNode::hash() const { 35.7 + if (EliminateNestedLocks) 35.8 + return NO_HASH; // Each locked region has own BoxLock node 35.9 return Node::hash() + _slot + (_is_eliminated ? Compile::current()->fixed_slots() : 0); 35.10 } 35.11 35.12 //------------------------------cmp-------------------------------------------- 35.13 uint BoxLockNode::cmp( const Node &n ) const { 35.14 + if (EliminateNestedLocks) 35.15 + return (&n == this); // Always fail except on self 35.16 const BoxLockNode &bn = (const BoxLockNode &)n; 35.17 return bn._slot == _slot && bn._is_eliminated == _is_eliminated; 35.18 } 35.19 35.20 -OptoReg::Name BoxLockNode::stack_slot(Node* box_node) { 35.21 - // Chase down the BoxNode 35.22 - while (!box_node->is_BoxLock()) { 35.23 +BoxLockNode* BoxLockNode::box_node(Node* box) { 35.24 + // Chase down the BoxNode after RA which may spill box nodes. 35.25 + while (!box->is_BoxLock()) { 35.26 // if (box_node->is_SpillCopy()) { 35.27 // Node *m = box_node->in(1); 35.28 // if (m->is_Mach() && m->as_Mach()->ideal_Opcode() == Op_StoreP) { 35.29 @@ -68,10 +72,64 @@ 35.30 // continue; 35.31 // } 35.32 // } 35.33 - assert(box_node->is_SpillCopy() || box_node->is_Phi(), "Bad spill of Lock."); 35.34 - box_node = box_node->in(1); 35.35 + assert(box->is_SpillCopy() || box->is_Phi(), "Bad spill of Lock."); 35.36 + // Only BoxLock nodes with the same stack slot are merged. 35.37 + // So it is enough to trace one path to find the slot value. 35.38 + box = box->in(1); 35.39 } 35.40 - return box_node->in_RegMask(0).find_first_elem(); 35.41 + return box->as_BoxLock(); 35.42 +} 35.43 + 35.44 +OptoReg::Name BoxLockNode::reg(Node* box) { 35.45 + return box_node(box)->in_RegMask(0).find_first_elem(); 35.46 +} 35.47 + 35.48 +// Is BoxLock node used for one simple lock region (same box and obj)? 35.49 +bool BoxLockNode::is_simple_lock_region(LockNode** unique_lock, Node* obj) { 35.50 + LockNode* lock = NULL; 35.51 + bool has_one_lock = false; 35.52 + for (uint i = 0; i < this->outcnt(); i++) { 35.53 + Node* n = this->raw_out(i); 35.54 + assert(!n->is_Phi(), "should not merge BoxLock nodes"); 35.55 + if (n->is_AbstractLock()) { 35.56 + AbstractLockNode* alock = n->as_AbstractLock(); 35.57 + // Check lock's box since box could be referenced by Lock's debug info. 35.58 + if (alock->box_node() == this) { 35.59 + if (alock->obj_node()->eqv_uncast(obj)) { 35.60 + if ((unique_lock != NULL) && alock->is_Lock()) { 35.61 + if (lock == NULL) { 35.62 + lock = alock->as_Lock(); 35.63 + has_one_lock = true; 35.64 + } else if (lock != alock->as_Lock()) { 35.65 + has_one_lock = false; 35.66 + } 35.67 + } 35.68 + } else { 35.69 + return false; // Different objects 35.70 + } 35.71 + } 35.72 + } 35.73 + } 35.74 +#ifdef ASSERT 35.75 + // Verify that FastLock and Safepoint reference only this lock region. 35.76 + for (uint i = 0; i < this->outcnt(); i++) { 35.77 + Node* n = this->raw_out(i); 35.78 + if (n->is_FastLock()) { 35.79 + FastLockNode* flock = n->as_FastLock(); 35.80 + assert((flock->box_node() == this) && flock->obj_node()->eqv_uncast(obj),""); 35.81 + } 35.82 + // Don't check monitor info in safepoints since the referenced object could 35.83 + // be different from the locked object. It could be Phi node of different 35.84 + // cast nodes which point to this locked object. 35.85 + // We assume that no other objects could be referenced in monitor info 35.86 + // associated with this BoxLock node because all associated locks and 35.87 + // unlocks are reference only this one object. 35.88 + } 35.89 +#endif 35.90 + if (unique_lock != NULL && has_one_lock) { 35.91 + *unique_lock = lock; 35.92 + } 35.93 + return true; 35.94 } 35.95 35.96 //=============================================================================
36.1 --- a/src/share/vm/opto/locknode.hpp Tue Jan 17 13:08:52 2012 -0500 36.2 +++ b/src/share/vm/opto/locknode.hpp Tue Jan 17 21:25:28 2012 -0500 36.3 @@ -49,11 +49,11 @@ 36.4 36.5 //------------------------------BoxLockNode------------------------------------ 36.6 class BoxLockNode : public Node { 36.7 + const int _slot; // stack slot 36.8 + RegMask _inmask; // OptoReg corresponding to stack slot 36.9 + bool _is_eliminated; // Associated locks were safely eliminated 36.10 + 36.11 public: 36.12 - const int _slot; 36.13 - RegMask _inmask; 36.14 - bool _is_eliminated; // indicates this lock was safely eliminated 36.15 - 36.16 BoxLockNode( int lock ); 36.17 virtual int Opcode() const; 36.18 virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const; 36.19 @@ -66,11 +66,19 @@ 36.20 virtual const class Type *bottom_type() const { return TypeRawPtr::BOTTOM; } 36.21 virtual uint ideal_reg() const { return Op_RegP; } 36.22 36.23 - static OptoReg::Name stack_slot(Node* box_node); 36.24 + static OptoReg::Name reg(Node* box_node); 36.25 + static BoxLockNode* box_node(Node* box_node); 36.26 + static bool same_slot(Node* box1, Node* box2) { 36.27 + return box1->as_BoxLock()->_slot == box2->as_BoxLock()->_slot; 36.28 + } 36.29 + int stack_slot() const { return _slot; } 36.30 36.31 - bool is_eliminated() { return _is_eliminated; } 36.32 + bool is_eliminated() const { return _is_eliminated; } 36.33 // mark lock as eliminated. 36.34 - void set_eliminated() { _is_eliminated = true; } 36.35 + void set_eliminated() { _is_eliminated = true; } 36.36 + 36.37 + // Is BoxLock node used for one simple lock region? 36.38 + bool is_simple_lock_region(LockNode** unique_lock, Node* obj); 36.39 36.40 #ifndef PRODUCT 36.41 virtual void format( PhaseRegAlloc *, outputStream *st ) const; 36.42 @@ -91,6 +99,7 @@ 36.43 } 36.44 Node* obj_node() const { return in(1); } 36.45 Node* box_node() const { return in(2); } 36.46 + void set_box_node(Node* box) { set_req(2, box); } 36.47 36.48 // FastLock and FastUnlockNode do not hash, we need one for each correspoding 36.49 // LockNode/UnLockNode to avoid creating Phi's.
37.1 --- a/src/share/vm/opto/loopnode.cpp Tue Jan 17 13:08:52 2012 -0500 37.2 +++ b/src/share/vm/opto/loopnode.cpp Tue Jan 17 21:25:28 2012 -0500 37.3 @@ -3278,16 +3278,7 @@ 37.4 #ifdef ASSERT 37.5 if (legal->is_Start() && !early->is_Root()) { 37.6 // Bad graph. Print idom path and fail. 37.7 - tty->print_cr( "Bad graph detected in build_loop_late"); 37.8 - tty->print("n: ");n->dump(); tty->cr(); 37.9 - tty->print("early: ");early->dump(); tty->cr(); 37.10 - int ct = 0; 37.11 - Node *dbg_legal = LCA; 37.12 - while(!dbg_legal->is_Start() && ct < 100) { 37.13 - tty->print("idom[%d] ",ct); dbg_legal->dump(); tty->cr(); 37.14 - ct++; 37.15 - dbg_legal = idom(dbg_legal); 37.16 - } 37.17 + dump_bad_graph(n, early, LCA); 37.18 assert(false, "Bad graph detected in build_loop_late"); 37.19 } 37.20 #endif 37.21 @@ -3337,6 +3328,88 @@ 37.22 chosen_loop->_body.push(n);// Collect inner loops 37.23 } 37.24 37.25 +#ifdef ASSERT 37.26 +void PhaseIdealLoop::dump_bad_graph(Node* n, Node* early, Node* LCA) { 37.27 + tty->print_cr( "Bad graph detected in build_loop_late"); 37.28 + tty->print("n: "); n->dump(); 37.29 + tty->print("early(n): "); early->dump(); 37.30 + if (n->in(0) != NULL && !n->in(0)->is_top() && 37.31 + n->in(0) != early && !n->in(0)->is_Root()) { 37.32 + tty->print("n->in(0): "); n->in(0)->dump(); 37.33 + } 37.34 + for (uint i = 1; i < n->req(); i++) { 37.35 + Node* in1 = n->in(i); 37.36 + if (in1 != NULL && in1 != n && !in1->is_top()) { 37.37 + tty->print("n->in(%d): ", i); in1->dump(); 37.38 + Node* in1_early = get_ctrl(in1); 37.39 + tty->print("early(n->in(%d)): ", i); in1_early->dump(); 37.40 + if (in1->in(0) != NULL && !in1->in(0)->is_top() && 37.41 + in1->in(0) != in1_early && !in1->in(0)->is_Root()) { 37.42 + tty->print("n->in(%d)->in(0): ", i); in1->in(0)->dump(); 37.43 + } 37.44 + for (uint j = 1; j < in1->req(); j++) { 37.45 + Node* in2 = in1->in(j); 37.46 + if (in2 != NULL && in2 != n && in2 != in1 && !in2->is_top()) { 37.47 + tty->print("n->in(%d)->in(%d): ", i, j); in2->dump(); 37.48 + Node* in2_early = get_ctrl(in2); 37.49 + tty->print("early(n->in(%d)->in(%d)): ", i, j); in2_early->dump(); 37.50 + if (in2->in(0) != NULL && !in2->in(0)->is_top() && 37.51 + in2->in(0) != in2_early && !in2->in(0)->is_Root()) { 37.52 + tty->print("n->in(%d)->in(%d)->in(0): ", i, j); in2->in(0)->dump(); 37.53 + } 37.54 + } 37.55 + } 37.56 + } 37.57 + } 37.58 + tty->cr(); 37.59 + tty->print("LCA(n): "); LCA->dump(); 37.60 + for (uint i = 0; i < n->outcnt(); i++) { 37.61 + Node* u1 = n->raw_out(i); 37.62 + if (u1 == n) 37.63 + continue; 37.64 + tty->print("n->out(%d): ", i); u1->dump(); 37.65 + if (u1->is_CFG()) { 37.66 + for (uint j = 0; j < u1->outcnt(); j++) { 37.67 + Node* u2 = u1->raw_out(j); 37.68 + if (u2 != u1 && u2 != n && u2->is_CFG()) { 37.69 + tty->print("n->out(%d)->out(%d): ", i, j); u2->dump(); 37.70 + } 37.71 + } 37.72 + } else { 37.73 + Node* u1_later = get_ctrl(u1); 37.74 + tty->print("later(n->out(%d)): ", i); u1_later->dump(); 37.75 + if (u1->in(0) != NULL && !u1->in(0)->is_top() && 37.76 + u1->in(0) != u1_later && !u1->in(0)->is_Root()) { 37.77 + tty->print("n->out(%d)->in(0): ", i); u1->in(0)->dump(); 37.78 + } 37.79 + for (uint j = 0; j < u1->outcnt(); j++) { 37.80 + Node* u2 = u1->raw_out(j); 37.81 + if (u2 == n || u2 == u1) 37.82 + continue; 37.83 + tty->print("n->out(%d)->out(%d): ", i, j); u2->dump(); 37.84 + if (!u2->is_CFG()) { 37.85 + Node* u2_later = get_ctrl(u2); 37.86 + tty->print("later(n->out(%d)->out(%d)): ", i, j); u2_later->dump(); 37.87 + if (u2->in(0) != NULL && !u2->in(0)->is_top() && 37.88 + u2->in(0) != u2_later && !u2->in(0)->is_Root()) { 37.89 + tty->print("n->out(%d)->in(0): ", i); u2->in(0)->dump(); 37.90 + } 37.91 + } 37.92 + } 37.93 + } 37.94 + } 37.95 + tty->cr(); 37.96 + int ct = 0; 37.97 + Node *dbg_legal = LCA; 37.98 + while(!dbg_legal->is_Start() && ct < 100) { 37.99 + tty->print("idom[%d] ",ct); dbg_legal->dump(); 37.100 + ct++; 37.101 + dbg_legal = idom(dbg_legal); 37.102 + } 37.103 + tty->cr(); 37.104 +} 37.105 +#endif 37.106 + 37.107 #ifndef PRODUCT 37.108 //------------------------------dump------------------------------------------- 37.109 void PhaseIdealLoop::dump( ) const {
38.1 --- a/src/share/vm/opto/loopnode.hpp Tue Jan 17 13:08:52 2012 -0500 38.2 +++ b/src/share/vm/opto/loopnode.hpp Tue Jan 17 21:25:28 2012 -0500 38.3 @@ -1040,6 +1040,10 @@ 38.4 bool created_loop_node() { return _created_loop_node; } 38.5 void register_new_node( Node *n, Node *blk ); 38.6 38.7 +#ifdef ASSERT 38.8 +void dump_bad_graph(Node* n, Node* early, Node* LCA); 38.9 +#endif 38.10 + 38.11 #ifndef PRODUCT 38.12 void dump( ) const; 38.13 void dump( IdealLoopTree *loop, uint rpo_idx, Node_List &rpo_list ) const;
39.1 --- a/src/share/vm/opto/loopopts.cpp Tue Jan 17 13:08:52 2012 -0500 39.2 +++ b/src/share/vm/opto/loopopts.cpp Tue Jan 17 21:25:28 2012 -0500 39.3 @@ -819,6 +819,8 @@ 39.4 if( iff->is_If() ) { // Classic split-if? 39.5 if( iff->in(0) != n_ctrl ) return; // Compare must be in same blk as if 39.6 } else if (iff->is_CMove()) { // Trying to split-up a CMOVE 39.7 + // Can't split CMove with different control edge. 39.8 + if (iff->in(0) != NULL && iff->in(0) != n_ctrl ) return; 39.9 if( get_ctrl(iff->in(2)) == n_ctrl || 39.10 get_ctrl(iff->in(3)) == n_ctrl ) 39.11 return; // Inputs not yet split-up 39.12 @@ -937,7 +939,7 @@ 39.13 } 39.14 bool did_break = (i < imax); // Did we break out of the previous loop? 39.15 if (!did_break && n->outcnt() > 1) { // All uses in outer loops! 39.16 - Node *late_load_ctrl; 39.17 + Node *late_load_ctrl = NULL; 39.18 if (n->is_Load()) { 39.19 // If n is a load, get and save the result from get_late_ctrl(), 39.20 // to be later used in calculating the control for n's clones.
40.1 --- a/src/share/vm/opto/macro.cpp Tue Jan 17 13:08:52 2012 -0500 40.2 +++ b/src/share/vm/opto/macro.cpp Tue Jan 17 21:25:28 2012 -0500 40.3 @@ -1789,7 +1789,8 @@ 40.4 slow_call_address); 40.5 } 40.6 40.7 -//-----------------------mark_eliminated_locking_nodes----------------------- 40.8 +//-------------------mark_eliminated_box---------------------------------- 40.9 +// 40.10 // During EA obj may point to several objects but after few ideal graph 40.11 // transformations (CCP) it may point to only one non escaping object 40.12 // (but still using phi), corresponding locks and unlocks will be marked 40.13 @@ -1800,62 +1801,148 @@ 40.14 // marked for elimination since new obj has no escape information. 40.15 // Mark all associated (same box and obj) lock and unlock nodes for 40.16 // elimination if some of them marked already. 40.17 -void PhaseMacroExpand::mark_eliminated_locking_nodes(AbstractLockNode *alock) { 40.18 - if (!alock->is_eliminated()) { 40.19 +void PhaseMacroExpand::mark_eliminated_box(Node* oldbox, Node* obj) { 40.20 + if (oldbox->as_BoxLock()->is_eliminated()) 40.21 + return; // This BoxLock node was processed already. 40.22 + 40.23 + // New implementation (EliminateNestedLocks) has separate BoxLock 40.24 + // node for each locked region so mark all associated locks/unlocks as 40.25 + // eliminated even if different objects are referenced in one locked region 40.26 + // (for example, OSR compilation of nested loop inside locked scope). 40.27 + if (EliminateNestedLocks || 40.28 + oldbox->as_BoxLock()->is_simple_lock_region(NULL, obj)) { 40.29 + // Box is used only in one lock region. Mark this box as eliminated. 40.30 + _igvn.hash_delete(oldbox); 40.31 + oldbox->as_BoxLock()->set_eliminated(); // This changes box's hash value 40.32 + _igvn.hash_insert(oldbox); 40.33 + 40.34 + for (uint i = 0; i < oldbox->outcnt(); i++) { 40.35 + Node* u = oldbox->raw_out(i); 40.36 + if (u->is_AbstractLock() && !u->as_AbstractLock()->is_non_esc_obj()) { 40.37 + AbstractLockNode* alock = u->as_AbstractLock(); 40.38 + // Check lock's box since box could be referenced by Lock's debug info. 40.39 + if (alock->box_node() == oldbox) { 40.40 + // Mark eliminated all related locks and unlocks. 40.41 + alock->set_non_esc_obj(); 40.42 + } 40.43 + } 40.44 + } 40.45 return; 40.46 } 40.47 - if (!alock->is_coarsened()) { // Eliminated by EA 40.48 - // Create new "eliminated" BoxLock node and use it 40.49 - // in monitor debug info for the same object. 40.50 - BoxLockNode* oldbox = alock->box_node()->as_BoxLock(); 40.51 - Node* obj = alock->obj_node(); 40.52 - if (!oldbox->is_eliminated()) { 40.53 - BoxLockNode* newbox = oldbox->clone()->as_BoxLock(); 40.54 + 40.55 + // Create new "eliminated" BoxLock node and use it in monitor debug info 40.56 + // instead of oldbox for the same object. 40.57 + BoxLockNode* newbox = oldbox->clone()->as_BoxLock(); 40.58 + 40.59 + // Note: BoxLock node is marked eliminated only here and it is used 40.60 + // to indicate that all associated lock and unlock nodes are marked 40.61 + // for elimination. 40.62 + newbox->set_eliminated(); 40.63 + transform_later(newbox); 40.64 + 40.65 + // Replace old box node with new box for all users of the same object. 40.66 + for (uint i = 0; i < oldbox->outcnt();) { 40.67 + bool next_edge = true; 40.68 + 40.69 + Node* u = oldbox->raw_out(i); 40.70 + if (u->is_AbstractLock()) { 40.71 + AbstractLockNode* alock = u->as_AbstractLock(); 40.72 + if (alock->box_node() == oldbox && alock->obj_node()->eqv_uncast(obj)) { 40.73 + // Replace Box and mark eliminated all related locks and unlocks. 40.74 + alock->set_non_esc_obj(); 40.75 + _igvn.hash_delete(alock); 40.76 + alock->set_box_node(newbox); 40.77 + _igvn._worklist.push(alock); 40.78 + next_edge = false; 40.79 + } 40.80 + } 40.81 + if (u->is_FastLock() && u->as_FastLock()->obj_node()->eqv_uncast(obj)) { 40.82 + FastLockNode* flock = u->as_FastLock(); 40.83 + assert(flock->box_node() == oldbox, "sanity"); 40.84 + _igvn.hash_delete(flock); 40.85 + flock->set_box_node(newbox); 40.86 + _igvn._worklist.push(flock); 40.87 + next_edge = false; 40.88 + } 40.89 + 40.90 + // Replace old box in monitor debug info. 40.91 + if (u->is_SafePoint() && u->as_SafePoint()->jvms()) { 40.92 + SafePointNode* sfn = u->as_SafePoint(); 40.93 + JVMState* youngest_jvms = sfn->jvms(); 40.94 + int max_depth = youngest_jvms->depth(); 40.95 + for (int depth = 1; depth <= max_depth; depth++) { 40.96 + JVMState* jvms = youngest_jvms->of_depth(depth); 40.97 + int num_mon = jvms->nof_monitors(); 40.98 + // Loop over monitors 40.99 + for (int idx = 0; idx < num_mon; idx++) { 40.100 + Node* obj_node = sfn->monitor_obj(jvms, idx); 40.101 + Node* box_node = sfn->monitor_box(jvms, idx); 40.102 + if (box_node == oldbox && obj_node->eqv_uncast(obj)) { 40.103 + int j = jvms->monitor_box_offset(idx); 40.104 + _igvn.hash_delete(u); 40.105 + u->set_req(j, newbox); 40.106 + _igvn._worklist.push(u); 40.107 + next_edge = false; 40.108 + } 40.109 + } 40.110 + } 40.111 + } 40.112 + if (next_edge) i++; 40.113 + } 40.114 +} 40.115 + 40.116 +//-----------------------mark_eliminated_locking_nodes----------------------- 40.117 +void PhaseMacroExpand::mark_eliminated_locking_nodes(AbstractLockNode *alock) { 40.118 + if (EliminateNestedLocks) { 40.119 + if (alock->is_nested()) { 40.120 + assert(alock->box_node()->as_BoxLock()->is_eliminated(), "sanity"); 40.121 + return; 40.122 + } else if (!alock->is_non_esc_obj()) { // Not eliminated or coarsened 40.123 + // Only Lock node has JVMState needed here. 40.124 + if (alock->jvms() != NULL && alock->as_Lock()->is_nested_lock_region()) { 40.125 + // Mark eliminated related nested locks and unlocks. 40.126 + Node* obj = alock->obj_node(); 40.127 + BoxLockNode* box_node = alock->box_node()->as_BoxLock(); 40.128 + assert(!box_node->is_eliminated(), "should not be marked yet"); 40.129 // Note: BoxLock node is marked eliminated only here 40.130 // and it is used to indicate that all associated lock 40.131 // and unlock nodes are marked for elimination. 40.132 - newbox->set_eliminated(); 40.133 - transform_later(newbox); 40.134 - // Replace old box node with new box for all users 40.135 - // of the same object. 40.136 - for (uint i = 0; i < oldbox->outcnt();) { 40.137 + box_node->set_eliminated(); // Box's hash is always NO_HASH here 40.138 + for (uint i = 0; i < box_node->outcnt(); i++) { 40.139 + Node* u = box_node->raw_out(i); 40.140 + if (u->is_AbstractLock()) { 40.141 + alock = u->as_AbstractLock(); 40.142 + if (alock->box_node() == box_node) { 40.143 + // Verify that this Box is referenced only by related locks. 40.144 + assert(alock->obj_node()->eqv_uncast(obj), ""); 40.145 + // Mark all related locks and unlocks. 40.146 + alock->set_nested(); 40.147 + } 40.148 + } 40.149 + } 40.150 + } 40.151 + return; 40.152 + } 40.153 + // Process locks for non escaping object 40.154 + assert(alock->is_non_esc_obj(), ""); 40.155 + } // EliminateNestedLocks 40.156 40.157 - bool next_edge = true; 40.158 - Node* u = oldbox->raw_out(i); 40.159 - if (u->is_AbstractLock() && 40.160 - u->as_AbstractLock()->obj_node() == obj && 40.161 - u->as_AbstractLock()->box_node() == oldbox) { 40.162 - // Mark all associated locks and unlocks. 40.163 - u->as_AbstractLock()->set_eliminated(); 40.164 - _igvn.hash_delete(u); 40.165 - u->set_req(TypeFunc::Parms + 1, newbox); 40.166 - next_edge = false; 40.167 - } 40.168 - // Replace old box in monitor debug info. 40.169 - if (u->is_SafePoint() && u->as_SafePoint()->jvms()) { 40.170 - SafePointNode* sfn = u->as_SafePoint(); 40.171 - JVMState* youngest_jvms = sfn->jvms(); 40.172 - int max_depth = youngest_jvms->depth(); 40.173 - for (int depth = 1; depth <= max_depth; depth++) { 40.174 - JVMState* jvms = youngest_jvms->of_depth(depth); 40.175 - int num_mon = jvms->nof_monitors(); 40.176 - // Loop over monitors 40.177 - for (int idx = 0; idx < num_mon; idx++) { 40.178 - Node* obj_node = sfn->monitor_obj(jvms, idx); 40.179 - Node* box_node = sfn->monitor_box(jvms, idx); 40.180 - if (box_node == oldbox && obj_node == obj) { 40.181 - int j = jvms->monitor_box_offset(idx); 40.182 - _igvn.hash_delete(u); 40.183 - u->set_req(j, newbox); 40.184 - next_edge = false; 40.185 - } 40.186 - } // for (int idx = 0; 40.187 - } // for (int depth = 1; 40.188 - } // if (u->is_SafePoint() 40.189 - if (next_edge) i++; 40.190 - } // for (uint i = 0; i < oldbox->outcnt();) 40.191 - } // if (!oldbox->is_eliminated()) 40.192 - } // if (!alock->is_coarsened()) 40.193 + if (alock->is_non_esc_obj()) { // Lock is used for non escaping object 40.194 + // Look for all locks of this object and mark them and 40.195 + // corresponding BoxLock nodes as eliminated. 40.196 + Node* obj = alock->obj_node(); 40.197 + for (uint j = 0; j < obj->outcnt(); j++) { 40.198 + Node* o = obj->raw_out(j); 40.199 + if (o->is_AbstractLock() && 40.200 + o->as_AbstractLock()->obj_node()->eqv_uncast(obj)) { 40.201 + alock = o->as_AbstractLock(); 40.202 + Node* box = alock->box_node(); 40.203 + // Replace old box node with new eliminated box for all users 40.204 + // of the same object and mark related locks as eliminated. 40.205 + mark_eliminated_box(box, obj); 40.206 + } 40.207 + } 40.208 + } 40.209 } 40.210 40.211 // we have determined that this lock/unlock can be eliminated, we simply 40.212 @@ -1870,7 +1957,7 @@ 40.213 return false; 40.214 } 40.215 #ifdef ASSERT 40.216 - if (alock->is_Lock() && !alock->is_coarsened()) { 40.217 + if (!alock->is_coarsened()) { 40.218 // Check that new "eliminated" BoxLock node is created. 40.219 BoxLockNode* oldbox = alock->box_node()->as_BoxLock(); 40.220 assert(oldbox->is_eliminated(), "should be done already"); 40.221 @@ -1962,6 +2049,8 @@ 40.222 Node* box = lock->box_node(); 40.223 Node* flock = lock->fastlock_node(); 40.224 40.225 + assert(!box->as_BoxLock()->is_eliminated(), "sanity"); 40.226 + 40.227 // Make the merge point 40.228 Node *region; 40.229 Node *mem_phi; 40.230 @@ -2196,6 +2285,8 @@ 40.231 Node* obj = unlock->obj_node(); 40.232 Node* box = unlock->box_node(); 40.233 40.234 + assert(!box->as_BoxLock()->is_eliminated(), "sanity"); 40.235 + 40.236 // No need for a null check on unlock 40.237 40.238 // Make the merge point
41.1 --- a/src/share/vm/opto/macro.hpp Tue Jan 17 13:08:52 2012 -0500 41.2 +++ b/src/share/vm/opto/macro.hpp Tue Jan 17 21:25:28 2012 -0500 41.3 @@ -92,6 +92,7 @@ 41.4 void process_users_of_allocation(AllocateNode *alloc); 41.5 41.6 void eliminate_card_mark(Node *cm); 41.7 + void mark_eliminated_box(Node* box, Node* obj); 41.8 void mark_eliminated_locking_nodes(AbstractLockNode *alock); 41.9 bool eliminate_locking_node(AbstractLockNode *alock); 41.10 void expand_lock_node(LockNode *lock);
42.1 --- a/src/share/vm/opto/memnode.cpp Tue Jan 17 13:08:52 2012 -0500 42.2 +++ b/src/share/vm/opto/memnode.cpp Tue Jan 17 21:25:28 2012 -0500 42.3 @@ -2201,7 +2201,7 @@ 42.4 // unsafe if I have intervening uses... Also disallowed for StoreCM 42.5 // since they must follow each StoreP operation. Redundant StoreCMs 42.6 // are eliminated just before matching in final_graph_reshape. 42.7 - if (mem->is_Store() && phase->eqv_uncast(mem->in(MemNode::Address), address) && 42.8 + if (mem->is_Store() && mem->in(MemNode::Address)->eqv_uncast(address) && 42.9 mem->Opcode() != Op_StoreCM) { 42.10 // Looking at a dead closed cycle of memory? 42.11 assert(mem != mem->in(MemNode::Memory), "dead loop in StoreNode::Ideal"); 42.12 @@ -2274,16 +2274,16 @@ 42.13 42.14 // Load then Store? Then the Store is useless 42.15 if (val->is_Load() && 42.16 - phase->eqv_uncast( val->in(MemNode::Address), adr ) && 42.17 - phase->eqv_uncast( val->in(MemNode::Memory ), mem ) && 42.18 + val->in(MemNode::Address)->eqv_uncast(adr) && 42.19 + val->in(MemNode::Memory )->eqv_uncast(mem) && 42.20 val->as_Load()->store_Opcode() == Opcode()) { 42.21 return mem; 42.22 } 42.23 42.24 // Two stores in a row of the same value? 42.25 if (mem->is_Store() && 42.26 - phase->eqv_uncast( mem->in(MemNode::Address), adr ) && 42.27 - phase->eqv_uncast( mem->in(MemNode::ValueIn), val ) && 42.28 + mem->in(MemNode::Address)->eqv_uncast(adr) && 42.29 + mem->in(MemNode::ValueIn)->eqv_uncast(val) && 42.30 mem->Opcode() == Opcode()) { 42.31 return mem; 42.32 }
43.1 --- a/src/share/vm/opto/node.cpp Tue Jan 17 13:08:52 2012 -0500 43.2 +++ b/src/share/vm/opto/node.cpp Tue Jan 17 21:25:28 2012 -0500 43.3 @@ -833,8 +833,20 @@ 43.4 43.5 //---------------------------uncast_helper------------------------------------- 43.6 Node* Node::uncast_helper(const Node* p) { 43.7 - uint max_depth = 3; 43.8 - for (uint i = 0; i < max_depth; i++) { 43.9 +#ifdef ASSERT 43.10 + uint depth_count = 0; 43.11 + const Node* orig_p = p; 43.12 +#endif 43.13 + 43.14 + while (true) { 43.15 +#ifdef ASSERT 43.16 + if (depth_count >= K) { 43.17 + orig_p->dump(4); 43.18 + if (p != orig_p) 43.19 + p->dump(1); 43.20 + } 43.21 + assert(depth_count++ < K, "infinite loop in Node::uncast_helper"); 43.22 +#endif 43.23 if (p == NULL || p->req() != 2) { 43.24 break; 43.25 } else if (p->is_ConstraintCast()) {
44.1 --- a/src/share/vm/opto/node.hpp Tue Jan 17 13:08:52 2012 -0500 44.2 +++ b/src/share/vm/opto/node.hpp Tue Jan 17 21:25:28 2012 -0500 44.3 @@ -429,6 +429,10 @@ 44.4 44.5 // Strip away casting. (It is depth-limited.) 44.6 Node* uncast() const; 44.7 + // Return whether two Nodes are equivalent, after stripping casting. 44.8 + bool eqv_uncast(const Node* n) const { 44.9 + return (this->uncast() == n->uncast()); 44.10 + } 44.11 44.12 private: 44.13 static Node* uncast_helper(const Node* n);
45.1 --- a/src/share/vm/opto/output.cpp Tue Jan 17 13:08:52 2012 -0500 45.2 +++ b/src/share/vm/opto/output.cpp Tue Jan 17 21:25:28 2012 -0500 45.3 @@ -924,10 +924,10 @@ 45.4 scval = new ConstantOopWriteValue(tp->is_oopptr()->const_oop()->constant_encoding()); 45.5 } 45.6 45.7 - OptoReg::Name box_reg = BoxLockNode::stack_slot(box_node); 45.8 + OptoReg::Name box_reg = BoxLockNode::reg(box_node); 45.9 Location basic_lock = Location::new_stk_loc(Location::normal,_regalloc->reg2offset(box_reg)); 45.10 - while( !box_node->is_BoxLock() ) box_node = box_node->in(1); 45.11 - monarray->append(new MonitorValue(scval, basic_lock, box_node->as_BoxLock()->is_eliminated())); 45.12 + bool eliminated = (box_node->is_BoxLock() && box_node->as_BoxLock()->is_eliminated()); 45.13 + monarray->append(new MonitorValue(scval, basic_lock, eliminated)); 45.14 } 45.15 45.16 // We dump the object pool first, since deoptimization reads it in first.
46.1 --- a/src/share/vm/opto/parse1.cpp Tue Jan 17 13:08:52 2012 -0500 46.2 +++ b/src/share/vm/opto/parse1.cpp Tue Jan 17 21:25:28 2012 -0500 46.3 @@ -1604,7 +1604,16 @@ 46.4 continue; 46.5 default: // All normal stuff 46.6 if (phi == NULL) { 46.7 - if (!check_elide_phi || !target->can_elide_SEL_phi(j)) { 46.8 + const JVMState* jvms = map()->jvms(); 46.9 + if (EliminateNestedLocks && 46.10 + jvms->is_mon(j) && jvms->is_monitor_box(j)) { 46.11 + // BoxLock nodes are not commoning. 46.12 + // Use old BoxLock node as merged box. 46.13 + assert(newin->jvms()->is_monitor_box(j), "sanity"); 46.14 + // This assert also tests that nodes are BoxLock. 46.15 + assert(BoxLockNode::same_slot(n, m), "sanity"); 46.16 + C->gvn_replace_by(n, m); 46.17 + } else if (!check_elide_phi || !target->can_elide_SEL_phi(j)) { 46.18 phi = ensure_phi(j, nophi); 46.19 } 46.20 }
47.1 --- a/src/share/vm/opto/phaseX.hpp Tue Jan 17 13:08:52 2012 -0500 47.2 +++ b/src/share/vm/opto/phaseX.hpp Tue Jan 17 21:25:28 2012 -0500 47.3 @@ -256,11 +256,6 @@ 47.4 // For pessimistic optimizations this is simply pointer equivalence. 47.5 bool eqv(const Node* n1, const Node* n2) const { return n1 == n2; } 47.6 47.7 - // Return whether two Nodes are equivalent, after stripping casting. 47.8 - bool eqv_uncast(const Node* n1, const Node* n2) const { 47.9 - return eqv(n1->uncast(), n2->uncast()); 47.10 - } 47.11 - 47.12 // For pessimistic passes, the return type must monotonically narrow. 47.13 // For optimistic passes, the return type must monotonically widen. 47.14 // It is possible to get into a "death march" in either type of pass,
48.1 --- a/src/share/vm/opto/postaloc.cpp Tue Jan 17 13:08:52 2012 -0500 48.2 +++ b/src/share/vm/opto/postaloc.cpp Tue Jan 17 21:25:28 2012 -0500 48.3 @@ -89,32 +89,62 @@ 48.4 return blk_adjust; 48.5 } 48.6 48.7 +#ifdef ASSERT 48.8 +static bool expected_yanked_node(Node *old, Node *orig_old) { 48.9 + // This code is expected only next original nodes: 48.10 + // - load from constant table node which may have next data input nodes: 48.11 + // MachConstantBase, Phi, MachTemp, MachSpillCopy 48.12 + // - load constant node which may have next data input nodes: 48.13 + // MachTemp, MachSpillCopy 48.14 + // - MachSpillCopy 48.15 + // - MachProj and Copy dead nodes 48.16 + if (old->is_MachSpillCopy()) { 48.17 + return true; 48.18 + } else if (old->is_Con()) { 48.19 + return true; 48.20 + } else if (old->is_MachProj()) { // Dead kills projection of Con node 48.21 + return (old == orig_old); 48.22 + } else if (old->is_Copy()) { // Dead copy of a callee-save value 48.23 + return (old == orig_old); 48.24 + } else if (old->is_MachTemp()) { 48.25 + return orig_old->is_Con(); 48.26 + } else if (old->is_Phi() || old->is_MachConstantBase()) { 48.27 + return (orig_old->is_Con() && orig_old->is_MachConstant()); 48.28 + } 48.29 + return false; 48.30 +} 48.31 +#endif 48.32 + 48.33 //------------------------------yank_if_dead----------------------------------- 48.34 -// Removed an edge from 'old'. Yank if dead. Return adjustment counts to 48.35 +// Removed edges from 'old'. Yank if dead. Return adjustment counts to 48.36 // iterators in the current block. 48.37 -int PhaseChaitin::yank_if_dead( Node *old, Block *current_block, Node_List *value, Node_List *regnd ) { 48.38 +int PhaseChaitin::yank_if_dead_recurse(Node *old, Node *orig_old, Block *current_block, 48.39 + Node_List *value, Node_List *regnd) { 48.40 int blk_adjust=0; 48.41 - while (old->outcnt() == 0 && old != C->top()) { 48.42 + if (old->outcnt() == 0 && old != C->top()) { 48.43 +#ifdef ASSERT 48.44 + if (!expected_yanked_node(old, orig_old)) { 48.45 + tty->print_cr("=============================================="); 48.46 + tty->print_cr("orig_old:"); 48.47 + orig_old->dump(); 48.48 + tty->print_cr("old:"); 48.49 + old->dump(); 48.50 + assert(false, "unexpected yanked node"); 48.51 + } 48.52 + if (old->is_Con()) 48.53 + orig_old = old; // Reset to satisfy expected nodes checks. 48.54 +#endif 48.55 blk_adjust += yank(old, current_block, value, regnd); 48.56 48.57 - Node *tmp = NULL; 48.58 for (uint i = 1; i < old->req(); i++) { 48.59 - if (old->in(i)->is_MachTemp()) { 48.60 - // handle TEMP inputs 48.61 - Node* machtmp = old->in(i); 48.62 - if (machtmp->outcnt() == 1) { 48.63 - assert(machtmp->unique_out() == old, "sanity"); 48.64 - blk_adjust += yank(machtmp, current_block, value, regnd); 48.65 - machtmp->disconnect_inputs(NULL); 48.66 - } 48.67 - } else { 48.68 - assert(tmp == NULL, "can't handle more non MachTemp inputs"); 48.69 - tmp = old->in(i); 48.70 + Node* n = old->in(i); 48.71 + if (n != NULL) { 48.72 + old->set_req(i, NULL); 48.73 + blk_adjust += yank_if_dead_recurse(n, orig_old, current_block, value, regnd); 48.74 } 48.75 } 48.76 + // Disconnect control and remove precedence edges if any exist 48.77 old->disconnect_inputs(NULL); 48.78 - if( !tmp ) break; 48.79 - old = tmp; 48.80 } 48.81 return blk_adjust; 48.82 }
49.1 --- a/src/share/vm/opto/subnode.cpp Tue Jan 17 13:08:52 2012 -0500 49.2 +++ b/src/share/vm/opto/subnode.cpp Tue Jan 17 21:25:28 2012 -0500 49.3 @@ -91,7 +91,7 @@ 49.4 49.5 // Not correct for SubFnode and AddFNode (must check for infinity) 49.6 // Equal? Subtract is zero 49.7 - if (phase->eqv_uncast(in1, in2)) return add_id(); 49.8 + if (in1->eqv_uncast(in2)) return add_id(); 49.9 49.10 // Either input is BOTTOM ==> the result is the local BOTTOM 49.11 if( t1 == Type::BOTTOM || t2 == Type::BOTTOM )
50.1 --- a/src/share/vm/runtime/arguments.cpp Tue Jan 17 13:08:52 2012 -0500 50.2 +++ b/src/share/vm/runtime/arguments.cpp Tue Jan 17 21:25:28 2012 -0500 50.3 @@ -3160,6 +3160,9 @@ 50.4 if (!UseBiasedLocking || EmitSync != 0) { 50.5 UseOptoBiasInlining = false; 50.6 } 50.7 + if (!EliminateLocks) { 50.8 + EliminateNestedLocks = false; 50.9 + } 50.10 #endif 50.11 50.12 if (PrintAssembly && FLAG_IS_DEFAULT(DebugNonSafepoints)) {
51.1 --- a/src/share/vm/runtime/deoptimization.cpp Tue Jan 17 13:08:52 2012 -0500 51.2 +++ b/src/share/vm/runtime/deoptimization.cpp Tue Jan 17 21:25:28 2012 -0500 51.3 @@ -211,7 +211,7 @@ 51.4 #ifdef COMPILER2 51.5 // Reallocate the non-escaping objects and restore their fields. Then 51.6 // relock objects if synchronization on them was eliminated. 51.7 - if (DoEscapeAnalysis) { 51.8 + if (DoEscapeAnalysis || EliminateNestedLocks) { 51.9 if (EliminateAllocations) { 51.10 assert (chunk->at(0)->scope() != NULL,"expect only compiled java frames"); 51.11 GrowableArray<ScopeValue*>* objects = chunk->at(0)->scope()->objects();
52.1 --- a/src/share/vm/runtime/vmStructs.cpp Tue Jan 17 13:08:52 2012 -0500 52.2 +++ b/src/share/vm/runtime/vmStructs.cpp Tue Jan 17 21:25:28 2012 -0500 52.3 @@ -307,7 +307,7 @@ 52.4 nonstatic_field(instanceKlass, _static_field_size, int) \ 52.5 nonstatic_field(instanceKlass, _static_oop_field_count, u2) \ 52.6 nonstatic_field(instanceKlass, _nonstatic_oop_map_size, int) \ 52.7 - nonstatic_field(instanceKlass, _misc_flags, u1) \ 52.8 + nonstatic_field(instanceKlass, _is_marked_dependent, bool) \ 52.9 nonstatic_field(instanceKlass, _minor_version, u2) \ 52.10 nonstatic_field(instanceKlass, _major_version, u2) \ 52.11 nonstatic_field(instanceKlass, _init_state, u1) \ 52.12 @@ -2386,7 +2386,6 @@ 52.13 declare_constant(instanceKlass::being_initialized) \ 52.14 declare_constant(instanceKlass::fully_initialized) \ 52.15 declare_constant(instanceKlass::initialization_error) \ 52.16 - declare_constant(instanceKlass::IS_MARKED_DEPENDENT) \ 52.17 \ 52.18 /*********************************/ \ 52.19 /* Symbol* - symbol max length */ \
53.1 --- a/test/compiler/7116216/StackOverflow.java Tue Jan 17 13:08:52 2012 -0500 53.2 +++ b/test/compiler/7116216/StackOverflow.java Tue Jan 17 21:25:28 2012 -0500 53.3 @@ -30,7 +30,7 @@ 53.4 * @run main/othervm -Xcomp -Xbatch StackOverflow 53.5 */ 53.6 53.7 -class StackOverflow { 53.8 +public class StackOverflow { 53.9 static String stackOverflow_largeFrame_liveOopForGC; 53.10 53.11 public static int stackOverflow_largeFrame(int call_count, String liveOopForGC) {
