Mercurial > jdk8-mips64-public > hotspot / changeset

     1.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Thu Jan 12 09:59:55 2012 -0500
     1.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Fri Jan 13 01:55:22 2012 -0800
     1.3 @@ -1,5 +1,5 @@
     1.4  /*
     1.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
     1.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
     1.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     1.8   *
     1.9   * This code is free software; you can redistribute it and/or modify it
    1.10 @@ -31,6 +31,7 @@
    1.11  #include "gc_implementation/g1/g1ErgoVerbose.hpp"
    1.12  #include "gc_implementation/g1/g1OopClosures.inline.hpp"
    1.13  #include "gc_implementation/g1/g1RemSet.hpp"
    1.14 +#include "gc_implementation/g1/heapRegion.inline.hpp"
    1.15  #include "gc_implementation/g1/heapRegionRemSet.hpp"
    1.16  #include "gc_implementation/g1/heapRegionSeq.inline.hpp"
    1.17  #include "gc_implementation/shared/vmGCOperations.hpp"
    1.18 @@ -183,12 +184,11 @@
    1.19  void CMMarkStack::allocate(size_t size) {
    1.20    _base = NEW_C_HEAP_ARRAY(oop, size);
    1.21    if (_base == NULL) {
    1.22 -    vm_exit_during_initialization("Failed to allocate "
    1.23 -                                  "CM region mark stack");
    1.24 +    vm_exit_during_initialization("Failed to allocate CM region mark stack");
    1.25    }
    1.26    _index = 0;
    1.27    _capacity = (jint) size;
    1.28 -  _oops_do_bound = -1;
    1.29 +  _saved_index = -1;
    1.30    NOT_PRODUCT(_max_depth = 0);
    1.31  }
    1.32  
    1.33 @@ -283,7 +283,6 @@
    1.34    }
    1.35  }
    1.36  
    1.37 -
    1.38  CMRegionStack::CMRegionStack() : _base(NULL) {}
    1.39  
    1.40  void CMRegionStack::allocate(size_t size) {
    1.41 @@ -302,6 +301,8 @@
    1.42  }
    1.43  
    1.44  void CMRegionStack::push_lock_free(MemRegion mr) {
    1.45 +  guarantee(false, "push_lock_free(): don't call this any more");
    1.46 +
    1.47    assert(mr.word_size() > 0, "Precondition");
    1.48    while (true) {
    1.49      jint index = _index;
    1.50 @@ -325,6 +326,8 @@
    1.51  // marking / remark phases. Should only be called in tandem with
    1.52  // other lock-free pops.
    1.53  MemRegion CMRegionStack::pop_lock_free() {
    1.54 +  guarantee(false, "pop_lock_free(): don't call this any more");
    1.55 +
    1.56    while (true) {
    1.57      jint index = _index;
    1.58  
    1.59 @@ -390,6 +393,8 @@
    1.60  #endif
    1.61  
    1.62  bool CMRegionStack::invalidate_entries_into_cset() {
    1.63 +  guarantee(false, "invalidate_entries_into_cset(): don't call this any more");
    1.64 +
    1.65    bool result = false;
    1.66    G1CollectedHeap* g1h = G1CollectedHeap::heap();
    1.67    for (int i = 0; i < _oops_do_bound; ++i) {
    1.68 @@ -438,14 +443,29 @@
    1.69    return res;
    1.70  }
    1.71  
    1.72 +void CMMarkStack::note_start_of_gc() {
    1.73 +  assert(_saved_index == -1,
    1.74 +         "note_start_of_gc()/end_of_gc() bracketed incorrectly");
    1.75 +  _saved_index = _index;
    1.76 +}
    1.77 +
    1.78 +void CMMarkStack::note_end_of_gc() {
    1.79 +  // This is intentionally a guarantee, instead of an assert. If we
    1.80 +  // accidentally add something to the mark stack during GC, it
    1.81 +  // will be a correctness issue so it's better if we crash. we'll
    1.82 +  // only check this once per GC anyway, so it won't be a performance
    1.83 +  // issue in any way.
    1.84 +  guarantee(_saved_index == _index,
    1.85 +            err_msg("saved index: %d index: %d", _saved_index, _index));
    1.86 +  _saved_index = -1;
    1.87 +}
    1.88 +
    1.89  void CMMarkStack::oops_do(OopClosure* f) {
    1.90 -  if (_index == 0) return;
    1.91 -  assert(_oops_do_bound != -1 && _oops_do_bound <= _index,
    1.92 -         "Bound must be set.");
    1.93 -  for (int i = 0; i < _oops_do_bound; i++) {
    1.94 +  assert(_saved_index == _index,
    1.95 +         err_msg("saved index: %d index: %d", _saved_index, _index));
    1.96 +  for (int i = 0; i < _index; i += 1) {
    1.97      f->do_oop(&_base[i]);
    1.98    }
    1.99 -  _oops_do_bound = -1;
   1.100  }
   1.101  
   1.102  bool ConcurrentMark::not_yet_marked(oop obj) const {
   1.103 @@ -783,7 +803,7 @@
   1.104  public:
   1.105    bool doHeapRegion(HeapRegion* r) {
   1.106      if (!r->continuesHumongous()) {
   1.107 -      r->note_start_of_marking(true);
   1.108 +      r->note_start_of_marking();
   1.109      }
   1.110      return false;
   1.111    }
   1.112 @@ -804,6 +824,10 @@
   1.113  
   1.114    // Initialise marking structures. This has to be done in a STW phase.
   1.115    reset();
   1.116 +
   1.117 +  // For each region note start of marking.
   1.118 +  NoteStartOfMarkHRClosure startcl;
   1.119 +  g1h->heap_region_iterate(&startcl);
   1.120  }
   1.121  
   1.122  
   1.123 @@ -818,10 +842,6 @@
   1.124    // every remark and we'll eventually not need to cause one.
   1.125    force_overflow_stw()->init();
   1.126  
   1.127 -  // For each region note start of marking.
   1.128 -  NoteStartOfMarkHRClosure startcl;
   1.129 -  g1h->heap_region_iterate(&startcl);
   1.130 -
   1.131    // Start Concurrent Marking weak-reference discovery.
   1.132    ReferenceProcessor* rp = g1h->ref_processor_cm();
   1.133    // enable ("weak") refs discovery
   1.134 @@ -946,22 +966,9 @@
   1.135  }
   1.136  #endif // !PRODUCT
   1.137  
   1.138 -void ConcurrentMark::grayRoot(oop p) {
   1.139 -  HeapWord* addr = (HeapWord*) p;
   1.140 -  // We can't really check against _heap_start and _heap_end, since it
   1.141 -  // is possible during an evacuation pause with piggy-backed
   1.142 -  // initial-mark that the committed space is expanded during the
   1.143 -  // pause without CM observing this change. So the assertions below
   1.144 -  // is a bit conservative; but better than nothing.
   1.145 -  assert(_g1h->g1_committed().contains(addr),
   1.146 -         "address should be within the heap bounds");
   1.147 -
   1.148 -  if (!_nextMarkBitMap->isMarked(addr)) {
   1.149 -    _nextMarkBitMap->parMark(addr);
   1.150 -  }
   1.151 -}
   1.152 -
   1.153  void ConcurrentMark::grayRegionIfNecessary(MemRegion mr) {
   1.154 +  guarantee(false, "grayRegionIfNecessary(): don't call this any more");
   1.155 +
   1.156    // The objects on the region have already been marked "in bulk" by
   1.157    // the caller. We only need to decide whether to push the region on
   1.158    // the region stack or not.
   1.159 @@ -1007,6 +1014,8 @@
   1.160  }
   1.161  
   1.162  void ConcurrentMark::markAndGrayObjectIfNecessary(oop p) {
   1.163 +  guarantee(false, "markAndGrayObjectIfNecessary(): don't call this any more");
   1.164 +
   1.165    // The object is not marked by the caller. We need to at least mark
   1.166    // it and maybe push in on the stack.
   1.167  
   1.168 @@ -1224,7 +1233,6 @@
   1.169                                         true /* expected_active */);
   1.170  
   1.171      if (VerifyDuringGC) {
   1.172 -
   1.173        HandleMark hm;  // handle scope
   1.174        gclog_or_tty->print(" VerifyDuringGC:(after)");
   1.175        Universe::heap()->prepare_for_verify();
   1.176 @@ -1879,10 +1887,6 @@
   1.177    double end = os::elapsedTime();
   1.178    _cleanup_times.add((end - start) * 1000.0);
   1.179  
   1.180 -  // G1CollectedHeap::heap()->print();
   1.181 -  // gclog_or_tty->print_cr("HEAP GC TIME STAMP : %d",
   1.182 -  // G1CollectedHeap::heap()->get_gc_time_stamp());
   1.183 -
   1.184    if (PrintGC || PrintGCDetails) {
   1.185      g1h->print_size_transition(gclog_or_tty,
   1.186                                 start_used_bytes,
   1.187 @@ -2669,6 +2673,8 @@
   1.188  }
   1.189  
   1.190  void ConcurrentMark::drainAllSATBBuffers() {
   1.191 +  guarantee(false, "drainAllSATBBuffers(): don't call this any more");
   1.192 +
   1.193    CMGlobalObjectClosure oc(this);
   1.194    SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
   1.195    satb_mq_set.set_closure(&oc);
   1.196 @@ -2687,12 +2693,6 @@
   1.197    assert(satb_mq_set.completed_buffers_num() == 0, "invariant");
   1.198  }
   1.199  
   1.200 -void ConcurrentMark::markPrev(oop p) {
   1.201 -  // Note we are overriding the read-only view of the prev map here, via
   1.202 -  // the cast.
   1.203 -  ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*)p);
   1.204 -}
   1.205 -
   1.206  void ConcurrentMark::clear(oop p) {
   1.207    assert(p != NULL && p->is_oop(), "expected an oop");
   1.208    HeapWord* addr = (HeapWord*)p;
   1.209 @@ -2702,13 +2702,21 @@
   1.210    _nextMarkBitMap->clear(addr);
   1.211  }
   1.212  
   1.213 -void ConcurrentMark::clearRangeBothMaps(MemRegion mr) {
   1.214 +void ConcurrentMark::clearRangePrevBitmap(MemRegion mr) {
   1.215    // Note we are overriding the read-only view of the prev map here, via
   1.216    // the cast.
   1.217    ((CMBitMap*)_prevMarkBitMap)->clearRange(mr);
   1.218 +}
   1.219 +
   1.220 +void ConcurrentMark::clearRangeNextBitmap(MemRegion mr) {
   1.221    _nextMarkBitMap->clearRange(mr);
   1.222  }
   1.223  
   1.224 +void ConcurrentMark::clearRangeBothBitmaps(MemRegion mr) {
   1.225 +  clearRangePrevBitmap(mr);
   1.226 +  clearRangeNextBitmap(mr);
   1.227 +}
   1.228 +
   1.229  HeapRegion*
   1.230  ConcurrentMark::claim_region(int task_num) {
   1.231    // "checkpoint" the finger
   1.232 @@ -2803,6 +2811,9 @@
   1.233  }
   1.234  
   1.235  bool ConcurrentMark::invalidate_aborted_regions_in_cset() {
   1.236 +  guarantee(false, "invalidate_aborted_regions_in_cset(): "
   1.237 +                   "don't call this any more");
   1.238 +
   1.239    bool result = false;
   1.240    for (int i = 0; i < (int)_max_task_num; ++i) {
   1.241      CMTask* the_task = _tasks[i];
   1.242 @@ -2854,24 +2865,135 @@
   1.243      // ...then over the contents of the all the task queues.
   1.244      queue->oops_do(cl);
   1.245    }
   1.246 -
   1.247 -  // Invalidate any entries, that are in the region stack, that
   1.248 -  // point into the collection set
   1.249 -  if (_regionStack.invalidate_entries_into_cset()) {
   1.250 -    // otherwise, any gray objects copied during the evacuation pause
   1.251 -    // might not be visited.
   1.252 -    assert(_should_gray_objects, "invariant");
   1.253 +}
   1.254 +
   1.255 +#ifndef PRODUCT
   1.256 +enum VerifyNoCSetOopsPhase {
   1.257 +  VerifyNoCSetOopsStack,
   1.258 +  VerifyNoCSetOopsQueues,
   1.259 +  VerifyNoCSetOopsSATBCompleted,
   1.260 +  VerifyNoCSetOopsSATBThread
   1.261 +};
   1.262 +
   1.263 +class VerifyNoCSetOopsClosure : public OopClosure, public ObjectClosure  {
   1.264 +private:
   1.265 +  G1CollectedHeap* _g1h;
   1.266 +  VerifyNoCSetOopsPhase _phase;
   1.267 +  int _info;
   1.268 +
   1.269 +  const char* phase_str() {
   1.270 +    switch (_phase) {
   1.271 +    case VerifyNoCSetOopsStack:         return "Stack";
   1.272 +    case VerifyNoCSetOopsQueues:        return "Queue";
   1.273 +    case VerifyNoCSetOopsSATBCompleted: return "Completed SATB Buffers";
   1.274 +    case VerifyNoCSetOopsSATBThread:    return "Thread SATB Buffers";
   1.275 +    default:                            ShouldNotReachHere();
   1.276 +    }
   1.277 +    return NULL;
   1.278    }
   1.279  
   1.280 -  // Invalidate any aborted regions, recorded in the individual CM
   1.281 -  // tasks, that point into the collection set.
   1.282 -  if (invalidate_aborted_regions_in_cset()) {
   1.283 -    // otherwise, any gray objects copied during the evacuation pause
   1.284 -    // might not be visited.
   1.285 -    assert(_should_gray_objects, "invariant");
   1.286 +  void do_object_work(oop obj) {
   1.287 +    guarantee(!_g1h->obj_in_cs(obj),
   1.288 +              err_msg("obj: "PTR_FORMAT" in CSet, phase: %s, info: %d",
   1.289 +                      (void*) obj, phase_str(), _info));
   1.290    }
   1.291  
   1.292 +public:
   1.293 +  VerifyNoCSetOopsClosure() : _g1h(G1CollectedHeap::heap()) { }
   1.294 +
   1.295 +  void set_phase(VerifyNoCSetOopsPhase phase, int info = -1) {
   1.296 +    _phase = phase;
   1.297 +    _info = info;
   1.298 +  }
   1.299 +
   1.300 +  virtual void do_oop(oop* p) {
   1.301 +    oop obj = oopDesc::load_decode_heap_oop(p);
   1.302 +    do_object_work(obj);
   1.303 +  }
   1.304 +
   1.305 +  virtual void do_oop(narrowOop* p) {
   1.306 +    // We should not come across narrow oops while scanning marking
   1.307 +    // stacks and SATB buffers.
   1.308 +    ShouldNotReachHere();
   1.309 +  }
   1.310 +
   1.311 +  virtual void do_object(oop obj) {
   1.312 +    do_object_work(obj);
   1.313 +  }
   1.314 +};
   1.315 +
   1.316 +void ConcurrentMark::verify_no_cset_oops(bool verify_stacks,
   1.317 +                                         bool verify_enqueued_buffers,
   1.318 +                                         bool verify_thread_buffers,
   1.319 +                                         bool verify_fingers) {
   1.320 +  assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint");
   1.321 +  if (!G1CollectedHeap::heap()->mark_in_progress()) {
   1.322 +    return;
   1.323 +  }
   1.324 +
   1.325 +  VerifyNoCSetOopsClosure cl;
   1.326 +
   1.327 +  if (verify_stacks) {
   1.328 +    // Verify entries on the global mark stack
   1.329 +    cl.set_phase(VerifyNoCSetOopsStack);
   1.330 +    _markStack.oops_do(&cl);
   1.331 +
   1.332 +    // Verify entries on the task queues
   1.333 +    for (int i = 0; i < (int) _max_task_num; i += 1) {
   1.334 +      cl.set_phase(VerifyNoCSetOopsQueues, i);
   1.335 +      OopTaskQueue* queue = _task_queues->queue(i);
   1.336 +      queue->oops_do(&cl);
   1.337 +    }
   1.338 +  }
   1.339 +
   1.340 +  SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
   1.341 +
   1.342 +  // Verify entries on the enqueued SATB buffers
   1.343 +  if (verify_enqueued_buffers) {
   1.344 +    cl.set_phase(VerifyNoCSetOopsSATBCompleted);
   1.345 +    satb_qs.iterate_completed_buffers_read_only(&cl);
   1.346 +  }
   1.347 +
   1.348 +  // Verify entries on the per-thread SATB buffers
   1.349 +  if (verify_thread_buffers) {
   1.350 +    cl.set_phase(VerifyNoCSetOopsSATBThread);
   1.351 +    satb_qs.iterate_thread_buffers_read_only(&cl);
   1.352 +  }
   1.353 +
   1.354 +  if (verify_fingers) {
   1.355 +    // Verify the global finger
   1.356 +    HeapWord* global_finger = finger();
   1.357 +    if (global_finger != NULL && global_finger < _heap_end) {
   1.358 +      // The global finger always points to a heap region boundary. We
   1.359 +      // use heap_region_containing_raw() to get the containing region
   1.360 +      // given that the global finger could be pointing to a free region
   1.361 +      // which subsequently becomes continues humongous. If that
   1.362 +      // happens, heap_region_containing() will return the bottom of the
   1.363 +      // corresponding starts humongous region and the check below will
   1.364 +      // not hold any more.
   1.365 +      HeapRegion* global_hr = _g1h->heap_region_containing_raw(global_finger);
   1.366 +      guarantee(global_finger == global_hr->bottom(),
   1.367 +                err_msg("global finger: "PTR_FORMAT" region: "HR_FORMAT,
   1.368 +                        global_finger, HR_FORMAT_PARAMS(global_hr)));
   1.369 +    }
   1.370 +
   1.371 +    // Verify the task fingers
   1.372 +    assert(parallel_marking_threads() <= _max_task_num, "sanity");
   1.373 +    for (int i = 0; i < (int) parallel_marking_threads(); i += 1) {
   1.374 +      CMTask* task = _tasks[i];
   1.375 +      HeapWord* task_finger = task->finger();
   1.376 +      if (task_finger != NULL && task_finger < _heap_end) {
   1.377 +        // See above note on the global finger verification.
   1.378 +        HeapRegion* task_hr = _g1h->heap_region_containing_raw(task_finger);
   1.379 +        guarantee(task_finger == task_hr->bottom() ||
   1.380 +                  !task_hr->in_collection_set(),
   1.381 +                  err_msg("task finger: "PTR_FORMAT" region: "HR_FORMAT,
   1.382 +                          task_finger, HR_FORMAT_PARAMS(task_hr)));
   1.383 +      }
   1.384 +    }
   1.385 +  }
   1.386  }
   1.387 +#endif // PRODUCT
   1.388  
   1.389  void ConcurrentMark::clear_marking_state(bool clear_overflow) {
   1.390    _markStack.setEmpty();
   1.391 @@ -3080,19 +3202,6 @@
   1.392    }
   1.393  };
   1.394  
   1.395 -class SetClaimValuesInCSetHRClosure: public HeapRegionClosure {
   1.396 -  jint _claim_value;
   1.397 -
   1.398 -public:
   1.399 -  SetClaimValuesInCSetHRClosure(jint claim_value) :
   1.400 -    _claim_value(claim_value) { }
   1.401 -
   1.402 -  bool doHeapRegion(HeapRegion* hr) {
   1.403 -    hr->set_claim_value(_claim_value);
   1.404 -    return false;
   1.405 -  }
   1.406 -};
   1.407 -
   1.408  class G1ParCompleteMarkInCSetTask: public AbstractGangTask {
   1.409  protected:
   1.410    G1CollectedHeap* _g1h;
   1.411 @@ -3112,6 +3221,9 @@
   1.412  };
   1.413  
   1.414  void ConcurrentMark::complete_marking_in_collection_set() {
   1.415 +  guarantee(false, "complete_marking_in_collection_set(): "
   1.416 +                   "don't call this any more");
   1.417 +
   1.418    G1CollectedHeap* g1h =  G1CollectedHeap::heap();
   1.419  
   1.420    if (!g1h->mark_in_progress()) {
   1.421 @@ -3135,9 +3247,8 @@
   1.422  
   1.423    assert(g1h->check_cset_heap_region_claim_values(HeapRegion::CompleteMarkCSetClaimValue), "sanity");
   1.424  
   1.425 -  // Now reset the claim values in the regions in the collection set.
   1.426 -  SetClaimValuesInCSetHRClosure set_cv_cl(HeapRegion::InitialClaimValue);
   1.427 -  g1h->collection_set_iterate(&set_cv_cl);
   1.428 +  // Reset the claim values in the regions in the collection set.
   1.429 +  g1h->reset_cset_heap_region_claim_values();
   1.430  
   1.431    assert(g1h->check_cset_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity");
   1.432  
   1.433 @@ -3160,6 +3271,8 @@
   1.434  // newCSet().
   1.435  
   1.436  void ConcurrentMark::newCSet() {
   1.437 +  guarantee(false, "newCSet(): don't call this any more");
   1.438 +
   1.439    if (!concurrent_marking_in_progress()) {
   1.440      // nothing to do if marking is not in progress
   1.441      return;
   1.442 @@ -3198,6 +3311,8 @@
   1.443  }
   1.444  
   1.445  void ConcurrentMark::registerCSetRegion(HeapRegion* hr) {
   1.446 +  guarantee(false, "registerCSetRegion(): don't call this any more");
   1.447 +
   1.448    if (!concurrent_marking_in_progress()) return;
   1.449  
   1.450    HeapWord* region_end = hr->end();
   1.451 @@ -3209,6 +3324,9 @@
   1.452  // Resets the region fields of active CMTasks whose values point
   1.453  // into the collection set.
   1.454  void ConcurrentMark::reset_active_task_region_fields_in_cset() {
   1.455 +  guarantee(false, "reset_active_task_region_fields_in_cset(): "
   1.456 +                   "don't call this any more");
   1.457 +
   1.458    assert(SafepointSynchronize::is_at_safepoint(), "should be in STW");
   1.459    assert(parallel_marking_threads() <= _max_task_num, "sanity");
   1.460  
   1.461 @@ -3919,6 +4037,10 @@
   1.462  }
   1.463  
   1.464  void CMTask::drain_region_stack(BitMapClosure* bc) {
   1.465 +  assert(_cm->region_stack_empty(), "region stack should be empty");
   1.466 +  assert(_aborted_region.is_empty(), "aborted region should be empty");
   1.467 +  return;
   1.468 +
   1.469    if (has_aborted()) return;
   1.470  
   1.471    assert(_region_finger == NULL,

     2.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Thu Jan 12 09:59:55 2012 -0500
     2.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Fri Jan 13 01:55:22 2012 -0800
     2.3 @@ -1,5 +1,5 @@
     2.4  /*
     2.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
     2.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
     2.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     2.8   *
     2.9   * This code is free software; you can redistribute it and/or modify it
    2.10 @@ -166,10 +166,10 @@
    2.11  // Ideally this should be GrowableArray<> just like MSC's marking stack(s).
    2.12  class CMMarkStack VALUE_OBJ_CLASS_SPEC {
    2.13    ConcurrentMark* _cm;
    2.14 -  oop*   _base;      // bottom of stack
    2.15 -  jint   _index;     // one more than last occupied index
    2.16 -  jint   _capacity;  // max #elements
    2.17 -  jint   _oops_do_bound;  // Number of elements to include in next iteration.
    2.18 +  oop*   _base;        // bottom of stack
    2.19 +  jint   _index;       // one more than last occupied index
    2.20 +  jint   _capacity;    // max #elements
    2.21 +  jint   _saved_index; // value of _index saved at start of GC
    2.22    NOT_PRODUCT(jint _max_depth;)  // max depth plumbed during run
    2.23  
    2.24    bool   _overflow;
    2.25 @@ -247,16 +247,12 @@
    2.26  
    2.27    void setEmpty()   { _index = 0; clear_overflow(); }
    2.28  
    2.29 -  // Record the current size; a subsequent "oops_do" will iterate only over
    2.30 -  // indices valid at the time of this call.
    2.31 -  void set_oops_do_bound(jint bound = -1) {
    2.32 -    if (bound == -1) {
    2.33 -      _oops_do_bound = _index;
    2.34 -    } else {
    2.35 -      _oops_do_bound = bound;
    2.36 -    }
    2.37 -  }
    2.38 -  jint oops_do_bound() { return _oops_do_bound; }
    2.39 +  // Record the current index.
    2.40 +  void note_start_of_gc();
    2.41 +
    2.42 +  // Make sure that we have not added any entries to the stack during GC.
    2.43 +  void note_end_of_gc();
    2.44 +
    2.45    // iterate over the oops in the mark stack, up to the bound recorded via
    2.46    // the call above.
    2.47    void oops_do(OopClosure* f);
    2.48 @@ -724,10 +720,9 @@
    2.49    // G1CollectedHeap
    2.50  
    2.51    // This notifies CM that a root during initial-mark needs to be
    2.52 -  // grayed and it's MT-safe. Currently, we just mark it. But, in the
    2.53 -  // future, we can experiment with pushing it on the stack and we can
    2.54 -  // do this without changing G1CollectedHeap.
    2.55 -  void grayRoot(oop p);
    2.56 +  // grayed. It is MT-safe.
    2.57 +  inline void grayRoot(oop obj, size_t word_size);
    2.58 +
    2.59    // It's used during evacuation pauses to gray a region, if
    2.60    // necessary, and it's MT-safe. It assumes that the caller has
    2.61    // marked any objects on that region. If _should_gray_objects is
    2.62 @@ -735,6 +730,7 @@
    2.63    // pushed on the region stack, if it is located below the global
    2.64    // finger, otherwise we do nothing.
    2.65    void grayRegionIfNecessary(MemRegion mr);
    2.66 +
    2.67    // It's used during evacuation pauses to mark and, if necessary,
    2.68    // gray a single object and it's MT-safe. It assumes the caller did
    2.69    // not mark the object. If _should_gray_objects is true and we're
    2.70 @@ -791,24 +787,40 @@
    2.71  
    2.72    // Mark in the previous bitmap.  NB: this is usually read-only, so use
    2.73    // this carefully!
    2.74 -  void markPrev(oop p);
    2.75 +  inline void markPrev(oop p);
    2.76 +  inline void markNext(oop p);
    2.77    void clear(oop p);
    2.78 -  // Clears marks for all objects in the given range, for both prev and
    2.79 -  // next bitmaps.  NB: the previous bitmap is usually read-only, so use
    2.80 -  // this carefully!
    2.81 -  void clearRangeBothMaps(MemRegion mr);
    2.82 +  // Clears marks for all objects in the given range, for the prev,
    2.83 +  // next, or both bitmaps.  NB: the previous bitmap is usually
    2.84 +  // read-only, so use this carefully!
    2.85 +  void clearRangePrevBitmap(MemRegion mr);
    2.86 +  void clearRangeNextBitmap(MemRegion mr);
    2.87 +  void clearRangeBothBitmaps(MemRegion mr);
    2.88  
    2.89 -  // Record the current top of the mark and region stacks; a
    2.90 -  // subsequent oops_do() on the mark stack and
    2.91 -  // invalidate_entries_into_cset() on the region stack will iterate
    2.92 -  // only over indices valid at the time of this call.
    2.93 -  void set_oops_do_bound() {
    2.94 -    _markStack.set_oops_do_bound();
    2.95 -    _regionStack.set_oops_do_bound();
    2.96 +  // Notify data structures that a GC has started.
    2.97 +  void note_start_of_gc() {
    2.98 +    _markStack.note_start_of_gc();
    2.99    }
   2.100 +
   2.101 +  // Notify data structures that a GC is finished.
   2.102 +  void note_end_of_gc() {
   2.103 +    _markStack.note_end_of_gc();
   2.104 +  }
   2.105 +
   2.106    // Iterate over the oops in the mark stack and all local queues. It
   2.107    // also calls invalidate_entries_into_cset() on the region stack.
   2.108    void oops_do(OopClosure* f);
   2.109 +
   2.110 +  // Verify that there are no CSet oops on the stacks (taskqueues /
   2.111 +  // global mark stack), enqueued SATB buffers, per-thread SATB
   2.112 +  // buffers, and fingers (global / per-task). The boolean parameters
   2.113 +  // decide which of the above data structures to verify. If marking
   2.114 +  // is not in progress, it's a no-op.
   2.115 +  void verify_no_cset_oops(bool verify_stacks,
   2.116 +                           bool verify_enqueued_buffers,
   2.117 +                           bool verify_thread_buffers,
   2.118 +                           bool verify_fingers) PRODUCT_RETURN;
   2.119 +
   2.120    // It is called at the end of an evacuation pause during marking so
   2.121    // that CM is notified of where the new end of the heap is. It
   2.122    // doesn't do anything if concurrent_marking_in_progress() is false,
   2.123 @@ -1166,6 +1178,7 @@
   2.124    // It keeps picking SATB buffers and processing them until no SATB
   2.125    // buffers are available.
   2.126    void drain_satb_buffers();
   2.127 +
   2.128    // It keeps popping regions from the region stack and processing
   2.129    // them until the region stack is empty.
   2.130    void drain_region_stack(BitMapClosure* closure);

     3.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp	Thu Jan 12 09:59:55 2012 -0500
     3.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp	Fri Jan 13 01:55:22 2012 -0800
     3.3 @@ -1,5 +1,5 @@
     3.4  /*
     3.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
     3.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
     3.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     3.8   *
     3.9   * This code is free software; you can redistribute it and/or modify it
    3.10 @@ -153,4 +153,46 @@
    3.11    }
    3.12  }
    3.13  
    3.14 +inline void ConcurrentMark::markPrev(oop p) {
    3.15 +  assert(!_prevMarkBitMap->isMarked((HeapWord*) p), "sanity");
    3.16 +  // Note we are overriding the read-only view of the prev map here, via
    3.17 +  // the cast.
    3.18 +  ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*) p);
    3.19 +}
    3.20 +
    3.21 +inline void ConcurrentMark::markNext(oop p) {
    3.22 +  assert(!_nextMarkBitMap->isMarked((HeapWord*) p), "sanity");
    3.23 +  _nextMarkBitMap->mark((HeapWord*) p);
    3.24 +}
    3.25 +
    3.26 +inline void ConcurrentMark::grayRoot(oop obj, size_t word_size) {
    3.27 +  HeapWord* addr = (HeapWord*) obj;
    3.28 +
    3.29 +  // Currently we don't do anything with word_size but we will use it
    3.30 +  // in the very near future in the liveness calculation piggy-backing
    3.31 +  // changes.
    3.32 +
    3.33 +#ifdef ASSERT
    3.34 +  HeapRegion* hr = _g1h->heap_region_containing(addr);
    3.35 +  assert(hr != NULL, "sanity");
    3.36 +  assert(!hr->is_survivor(), "should not allocate survivors during IM");
    3.37 +  assert(addr < hr->next_top_at_mark_start(),
    3.38 +         err_msg("addr: "PTR_FORMAT" hr: "HR_FORMAT" NTAMS: "PTR_FORMAT,
    3.39 +                 addr, HR_FORMAT_PARAMS(hr), hr->next_top_at_mark_start()));
    3.40 +  // We cannot assert that word_size == obj->size() given that obj
    3.41 +  // might not be in a consistent state (another thread might be in
    3.42 +  // the process of copying it). So the best thing we can do is to
    3.43 +  // assert that word_size is under an upper bound which is its
    3.44 +  // containing region's capacity.
    3.45 +  assert(word_size * HeapWordSize <= hr->capacity(),
    3.46 +         err_msg("size: "SIZE_FORMAT" capacity: "SIZE_FORMAT" "HR_FORMAT,
    3.47 +                 word_size * HeapWordSize, hr->capacity(),
    3.48 +                 HR_FORMAT_PARAMS(hr)));
    3.49 +#endif // ASSERT
    3.50 +
    3.51 +  if (!_nextMarkBitMap->isMarked(addr)) {
    3.52 +    _nextMarkBitMap->parMark(addr);
    3.53 +  }
    3.54 +}
    3.55 +
    3.56  #endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP

     4.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Thu Jan 12 09:59:55 2012 -0500
     4.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Fri Jan 13 01:55:22 2012 -0800
     4.3 @@ -1,5 +1,5 @@
     4.4  /*
     4.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
     4.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
     4.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     4.8   *
     4.9   * This code is free software; you can redistribute it and/or modify it
    4.10 @@ -32,9 +32,11 @@
    4.11  #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
    4.12  #include "gc_implementation/g1/g1CollectorPolicy.hpp"
    4.13  #include "gc_implementation/g1/g1ErgoVerbose.hpp"
    4.14 +#include "gc_implementation/g1/g1EvacFailure.hpp"
    4.15  #include "gc_implementation/g1/g1MarkSweep.hpp"
    4.16  #include "gc_implementation/g1/g1OopClosures.inline.hpp"
    4.17  #include "gc_implementation/g1/g1RemSet.inline.hpp"
    4.18 +#include "gc_implementation/g1/heapRegion.inline.hpp"
    4.19  #include "gc_implementation/g1/heapRegionRemSet.hpp"
    4.20  #include "gc_implementation/g1/heapRegionSeq.inline.hpp"
    4.21  #include "gc_implementation/g1/vm_operations_g1.hpp"
    4.22 @@ -591,17 +593,29 @@
    4.23      }
    4.24      res = new_region_try_secondary_free_list();
    4.25    }
    4.26 -  if (res == NULL && do_expand) {
    4.27 +  if (res == NULL && do_expand && _expand_heap_after_alloc_failure) {
    4.28 +    // Currently, only attempts to allocate GC alloc regions set
    4.29 +    // do_expand to true. So, we should only reach here during a
    4.30 +    // safepoint. If this assumption changes we might have to
    4.31 +    // reconsider the use of _expand_heap_after_alloc_failure.
    4.32 +    assert(SafepointSynchronize::is_at_safepoint(), "invariant");
    4.33 +
    4.34      ergo_verbose1(ErgoHeapSizing,
    4.35                    "attempt heap expansion",
    4.36                    ergo_format_reason("region allocation request failed")
    4.37                    ergo_format_byte("allocation request"),
    4.38                    word_size * HeapWordSize);
    4.39      if (expand(word_size * HeapWordSize)) {
    4.40 -      // Even though the heap was expanded, it might not have reached
    4.41 -      // the desired size. So, we cannot assume that the allocation
    4.42 -      // will succeed.
    4.43 +      // Given that expand() succeeded in expanding the heap, and we
    4.44 +      // always expand the heap by an amount aligned to the heap
    4.45 +      // region size, the free list should in theory not be empty. So
    4.46 +      // it would probably be OK to use remove_head(). But the extra
    4.47 +      // check for NULL is unlikely to be a performance issue here (we
    4.48 +      // just expanded the heap!) so let's just be conservative and
    4.49 +      // use remove_head_or_null().
    4.50        res = _free_list.remove_head_or_null();
    4.51 +    } else {
    4.52 +      _expand_heap_after_alloc_failure = false;
    4.53      }
    4.54    }
    4.55    return res;
    4.56 @@ -1838,6 +1852,7 @@
    4.57    _young_list(new YoungList(this)),
    4.58    _gc_time_stamp(0),
    4.59    _retained_old_gc_alloc_region(NULL),
    4.60 +  _expand_heap_after_alloc_failure(true),
    4.61    _surviving_young_words(NULL),
    4.62    _full_collections_completed(0),
    4.63    _in_cset_fast_test(NULL),
    4.64 @@ -2605,12 +2620,16 @@
    4.65    }
    4.66  };
    4.67  
    4.68 -void
    4.69 -G1CollectedHeap::reset_heap_region_claim_values() {
    4.70 +void G1CollectedHeap::reset_heap_region_claim_values() {
    4.71    ResetClaimValuesClosure blk;
    4.72    heap_region_iterate(&blk);
    4.73  }
    4.74  
    4.75 +void G1CollectedHeap::reset_cset_heap_region_claim_values() {
    4.76 +  ResetClaimValuesClosure blk;
    4.77 +  collection_set_iterate(&blk);
    4.78 +}
    4.79 +
    4.80  #ifdef ASSERT
    4.81  // This checks whether all regions in the heap have the correct claim
    4.82  // value. I also piggy-backed on this a check to ensure that the
    4.83 @@ -3000,14 +3019,20 @@
    4.84        } else {
    4.85          VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo);
    4.86          r->object_iterate(&not_dead_yet_cl);
    4.87 -        if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
    4.88 -          gclog_or_tty->print_cr("["PTR_FORMAT","PTR_FORMAT"] "
    4.89 -                                 "max_live_bytes "SIZE_FORMAT" "
    4.90 -                                 "< calculated "SIZE_FORMAT,
    4.91 -                                 r->bottom(), r->end(),
    4.92 -                                 r->max_live_bytes(),
    4.93 +        if (_vo != VerifyOption_G1UseNextMarking) {
    4.94 +          if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
    4.95 +            gclog_or_tty->print_cr("["PTR_FORMAT","PTR_FORMAT"] "
    4.96 +                                   "max_live_bytes "SIZE_FORMAT" "
    4.97 +                                   "< calculated "SIZE_FORMAT,
    4.98 +                                   r->bottom(), r->end(),
    4.99 +                                   r->max_live_bytes(),
   4.100                                   not_dead_yet_cl.live_bytes());
   4.101 -          _failures = true;
   4.102 +            _failures = true;
   4.103 +          }
   4.104 +        } else {
   4.105 +          // When vo == UseNextMarking we cannot currently do a sanity
   4.106 +          // check on the live bytes as the calculation has not been
   4.107 +          // finalized yet.
   4.108          }
   4.109        }
   4.110      }
   4.111 @@ -3641,25 +3666,6 @@
   4.112          }
   4.113          perm_gen()->save_marks();
   4.114  
   4.115 -        // We must do this before any possible evacuation that should propagate
   4.116 -        // marks.
   4.117 -        if (mark_in_progress()) {
   4.118 -          double start_time_sec = os::elapsedTime();
   4.119 -
   4.120 -          _cm->drainAllSATBBuffers();
   4.121 -          double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0;
   4.122 -          g1_policy()->record_satb_drain_time(finish_mark_ms);
   4.123 -        }
   4.124 -        // Record the number of elements currently on the mark stack, so we
   4.125 -        // only iterate over these.  (Since evacuation may add to the mark
   4.126 -        // stack, doing more exposes race conditions.)  If no mark is in
   4.127 -        // progress, this will be zero.
   4.128 -        _cm->set_oops_do_bound();
   4.129 -
   4.130 -        if (mark_in_progress()) {
   4.131 -          concurrent_mark()->newCSet();
   4.132 -        }
   4.133 -
   4.134  #if YOUNG_LIST_VERBOSE
   4.135          gclog_or_tty->print_cr("\nBefore choosing collection set.\nYoung_list:");
   4.136          _young_list->print();
   4.137 @@ -3668,6 +3674,16 @@
   4.138  
   4.139          g1_policy()->choose_collection_set(target_pause_time_ms);
   4.140  
   4.141 +        _cm->note_start_of_gc();
   4.142 +        // We should not verify the per-thread SATB buffers given that
   4.143 +        // we have not filtered them yet (we'll do so during the
   4.144 +        // GC). We also call this after choose_collection_set() to
   4.145 +        // ensure that the CSet has been finalized.
   4.146 +        _cm->verify_no_cset_oops(true  /* verify_stacks */,
   4.147 +                                 true  /* verify_enqueued_buffers */,
   4.148 +                                 false /* verify_thread_buffers */,
   4.149 +                                 true  /* verify_fingers */);
   4.150 +
   4.151          if (_hr_printer.is_active()) {
   4.152            HeapRegion* hr = g1_policy()->collection_set();
   4.153            while (hr != NULL) {
   4.154 @@ -3684,16 +3700,6 @@
   4.155            }
   4.156          }
   4.157  
   4.158 -        // We have chosen the complete collection set. If marking is
   4.159 -        // active then, we clear the region fields of any of the
   4.160 -        // concurrent marking tasks whose region fields point into
   4.161 -        // the collection set as these values will become stale. This
   4.162 -        // will cause the owning marking threads to claim a new region
   4.163 -        // when marking restarts.
   4.164 -        if (mark_in_progress()) {
   4.165 -          concurrent_mark()->reset_active_task_region_fields_in_cset();
   4.166 -        }
   4.167 -
   4.168  #ifdef ASSERT
   4.169          VerifyCSetClosure cl;
   4.170          collection_set_iterate(&cl);
   4.171 @@ -3707,6 +3713,16 @@
   4.172          // Actually do the work...
   4.173          evacuate_collection_set();
   4.174  
   4.175 +        // We do this to mainly verify the per-thread SATB buffers
   4.176 +        // (which have been filtered by now) since we didn't verify
   4.177 +        // them earlier. No point in re-checking the stacks / enqueued
   4.178 +        // buffers given that the CSet has not changed since last time
   4.179 +        // we checked.
   4.180 +        _cm->verify_no_cset_oops(false /* verify_stacks */,
   4.181 +                                 false /* verify_enqueued_buffers */,
   4.182 +                                 true  /* verify_thread_buffers */,
   4.183 +                                 true  /* verify_fingers */);
   4.184 +
   4.185          free_collection_set(g1_policy()->collection_set());
   4.186          g1_policy()->clear_collection_set();
   4.187  
   4.188 @@ -3775,6 +3791,8 @@
   4.189            size_t expand_bytes = g1_policy()->expansion_amount();
   4.190            if (expand_bytes > 0) {
   4.191              size_t bytes_before = capacity();
   4.192 +            // No need for an ergo verbose message here,
   4.193 +            // expansion_amount() does this when it returns a value > 0.
   4.194              if (!expand(expand_bytes)) {
   4.195                // We failed to expand the heap so let's verify that
   4.196                // committed/uncommitted amount match the backing store
   4.197 @@ -3784,6 +3802,14 @@
   4.198            }
   4.199          }
   4.200  
   4.201 +        // We redo the verificaiton but now wrt to the new CSet which
   4.202 +        // has just got initialized after the previous CSet was freed.
   4.203 +        _cm->verify_no_cset_oops(true  /* verify_stacks */,
   4.204 +                                 true  /* verify_enqueued_buffers */,
   4.205 +                                 true  /* verify_thread_buffers */,
   4.206 +                                 true  /* verify_fingers */);
   4.207 +        _cm->note_end_of_gc();
   4.208 +
   4.209          double end_time_sec = os::elapsedTime();
   4.210          double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS;
   4.211          g1_policy()->record_pause_time_ms(pause_time_ms);
   4.212 @@ -3831,21 +3857,6 @@
   4.213          // CM reference discovery will be re-enabled if necessary.
   4.214        }
   4.215  
   4.216 -      {
   4.217 -        size_t expand_bytes = g1_policy()->expansion_amount();
   4.218 -        if (expand_bytes > 0) {
   4.219 -          size_t bytes_before = capacity();
   4.220 -          // No need for an ergo verbose message here,
   4.221 -          // expansion_amount() does this when it returns a value > 0.
   4.222 -          if (!expand(expand_bytes)) {
   4.223 -            // We failed to expand the heap so let's verify that
   4.224 -            // committed/uncommitted amount match the backing store
   4.225 -            assert(capacity() == _g1_storage.committed_size(), "committed size mismatch");
   4.226 -            assert(max_capacity() == _g1_storage.reserved_size(), "reserved size mismatch");
   4.227 -          }
   4.228 -        }
   4.229 -      }
   4.230 -
   4.231        // We should do this after we potentially expand the heap so
   4.232        // that all the COMMIT events are generated before the end GC
   4.233        // event, and after we retire the GC alloc regions so that all
   4.234 @@ -3949,6 +3960,8 @@
   4.235      // we allocate to in the region sets. We'll re-add it later, when
   4.236      // it's retired again.
   4.237      _old_set.remove(retained_region);
   4.238 +    bool during_im = g1_policy()->during_initial_mark_pause();
   4.239 +    retained_region->note_start_of_copying(during_im);
   4.240      _old_gc_alloc_region.set(retained_region);
   4.241      _hr_printer.reuse(retained_region);
   4.242    }
   4.243 @@ -3985,157 +3998,26 @@
   4.244    _evac_failure_scan_stack = NULL;
   4.245  }
   4.246  
   4.247 -class UpdateRSetDeferred : public OopsInHeapRegionClosure {
   4.248 -private:
   4.249 -  G1CollectedHeap* _g1;
   4.250 -  DirtyCardQueue *_dcq;
   4.251 -  CardTableModRefBS* _ct_bs;
   4.252 -
   4.253 -public:
   4.254 -  UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
   4.255 -    _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {}
   4.256 -
   4.257 -  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
   4.258 -  virtual void do_oop(      oop* p) { do_oop_work(p); }
   4.259 -  template <class T> void do_oop_work(T* p) {
   4.260 -    assert(_from->is_in_reserved(p), "paranoia");
   4.261 -    if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) &&
   4.262 -        !_from->is_survivor()) {
   4.263 -      size_t card_index = _ct_bs->index_for(p);
   4.264 -      if (_ct_bs->mark_card_deferred(card_index)) {
   4.265 -        _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
   4.266 -      }
   4.267 -    }
   4.268 -  }
   4.269 -};
   4.270 -
   4.271 -class RemoveSelfPointerClosure: public ObjectClosure {
   4.272 -private:
   4.273 -  G1CollectedHeap* _g1;
   4.274 -  ConcurrentMark* _cm;
   4.275 -  HeapRegion* _hr;
   4.276 -  size_t _prev_marked_bytes;
   4.277 -  size_t _next_marked_bytes;
   4.278 -  OopsInHeapRegionClosure *_cl;
   4.279 -public:
   4.280 -  RemoveSelfPointerClosure(G1CollectedHeap* g1, HeapRegion* hr,
   4.281 -                           OopsInHeapRegionClosure* cl) :
   4.282 -    _g1(g1), _hr(hr), _cm(_g1->concurrent_mark()),  _prev_marked_bytes(0),
   4.283 -    _next_marked_bytes(0), _cl(cl) {}
   4.284 -
   4.285 -  size_t prev_marked_bytes() { return _prev_marked_bytes; }
   4.286 -  size_t next_marked_bytes() { return _next_marked_bytes; }
   4.287 -
   4.288 -  // <original comment>
   4.289 -  // The original idea here was to coalesce evacuated and dead objects.
   4.290 -  // However that caused complications with the block offset table (BOT).
   4.291 -  // In particular if there were two TLABs, one of them partially refined.
   4.292 -  // |----- TLAB_1--------|----TLAB_2-~~~(partially refined part)~~~|
   4.293 -  // The BOT entries of the unrefined part of TLAB_2 point to the start
   4.294 -  // of TLAB_2. If the last object of the TLAB_1 and the first object
   4.295 -  // of TLAB_2 are coalesced, then the cards of the unrefined part
   4.296 -  // would point into middle of the filler object.
   4.297 -  // The current approach is to not coalesce and leave the BOT contents intact.
   4.298 -  // </original comment>
   4.299 -  //
   4.300 -  // We now reset the BOT when we start the object iteration over the
   4.301 -  // region and refine its entries for every object we come across. So
   4.302 -  // the above comment is not really relevant and we should be able
   4.303 -  // to coalesce dead objects if we want to.
   4.304 -  void do_object(oop obj) {
   4.305 -    HeapWord* obj_addr = (HeapWord*) obj;
   4.306 -    assert(_hr->is_in(obj_addr), "sanity");
   4.307 -    size_t obj_size = obj->size();
   4.308 -    _hr->update_bot_for_object(obj_addr, obj_size);
   4.309 -    if (obj->is_forwarded() && obj->forwardee() == obj) {
   4.310 -      // The object failed to move.
   4.311 -      assert(!_g1->is_obj_dead(obj), "We should not be preserving dead objs.");
   4.312 -      _cm->markPrev(obj);
   4.313 -      assert(_cm->isPrevMarked(obj), "Should be marked!");
   4.314 -      _prev_marked_bytes += (obj_size * HeapWordSize);
   4.315 -      if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) {
   4.316 -        _cm->markAndGrayObjectIfNecessary(obj);
   4.317 -      }
   4.318 -      obj->set_mark(markOopDesc::prototype());
   4.319 -      // While we were processing RSet buffers during the
   4.320 -      // collection, we actually didn't scan any cards on the
   4.321 -      // collection set, since we didn't want to update remebered
   4.322 -      // sets with entries that point into the collection set, given
   4.323 -      // that live objects fromthe collection set are about to move
   4.324 -      // and such entries will be stale very soon. This change also
   4.325 -      // dealt with a reliability issue which involved scanning a
   4.326 -      // card in the collection set and coming across an array that
   4.327 -      // was being chunked and looking malformed. The problem is
   4.328 -      // that, if evacuation fails, we might have remembered set
   4.329 -      // entries missing given that we skipped cards on the
   4.330 -      // collection set. So, we'll recreate such entries now.
   4.331 -      obj->oop_iterate(_cl);
   4.332 -      assert(_cm->isPrevMarked(obj), "Should be marked!");
   4.333 -    } else {
   4.334 -      // The object has been either evacuated or is dead. Fill it with a
   4.335 -      // dummy object.
   4.336 -      MemRegion mr((HeapWord*)obj, obj_size);
   4.337 -      CollectedHeap::fill_with_object(mr);
   4.338 -      _cm->clearRangeBothMaps(mr);
   4.339 -    }
   4.340 -  }
   4.341 -};
   4.342 -
   4.343  void G1CollectedHeap::remove_self_forwarding_pointers() {
   4.344 -  UpdateRSetImmediate immediate_update(_g1h->g1_rem_set());
   4.345 -  DirtyCardQueue dcq(&_g1h->dirty_card_queue_set());
   4.346 -  UpdateRSetDeferred deferred_update(_g1h, &dcq);
   4.347 -  OopsInHeapRegionClosure *cl;
   4.348 -  if (G1DeferredRSUpdate) {
   4.349 -    cl = &deferred_update;
   4.350 +  assert(check_cset_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity");
   4.351 +  assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!");
   4.352 +
   4.353 +  G1ParRemoveSelfForwardPtrsTask rsfp_task(this);
   4.354 +
   4.355 +  if (G1CollectedHeap::use_parallel_gc_threads()) {
   4.356 +    set_par_threads();
   4.357 +    workers()->run_task(&rsfp_task);
   4.358 +    set_par_threads(0);
   4.359    } else {
   4.360 -    cl = &immediate_update;
   4.361 -  }
   4.362 -  HeapRegion* cur = g1_policy()->collection_set();
   4.363 -  while (cur != NULL) {
   4.364 -    assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!");
   4.365 -    assert(!cur->isHumongous(), "sanity");
   4.366 -
   4.367 -    if (cur->evacuation_failed()) {
   4.368 -      assert(cur->in_collection_set(), "bad CS");
   4.369 -      RemoveSelfPointerClosure rspc(_g1h, cur, cl);
   4.370 -
   4.371 -      // In the common case we make sure that this is done when the
   4.372 -      // region is freed so that it is "ready-to-go" when it's
   4.373 -      // re-allocated. However, when evacuation failure happens, a
   4.374 -      // region will remain in the heap and might ultimately be added
   4.375 -      // to a CSet in the future. So we have to be careful here and
   4.376 -      // make sure the region's RSet is ready for parallel iteration
   4.377 -      // whenever this might be required in the future.
   4.378 -      cur->rem_set()->reset_for_par_iteration();
   4.379 -      cur->reset_bot();
   4.380 -      cl->set_region(cur);
   4.381 -      cur->object_iterate(&rspc);
   4.382 -
   4.383 -      // A number of manipulations to make the TAMS be the current top,
   4.384 -      // and the marked bytes be the ones observed in the iteration.
   4.385 -      if (_g1h->concurrent_mark()->at_least_one_mark_complete()) {
   4.386 -        // The comments below are the postconditions achieved by the
   4.387 -        // calls.  Note especially the last such condition, which says that
   4.388 -        // the count of marked bytes has been properly restored.
   4.389 -        cur->note_start_of_marking(false);
   4.390 -        // _next_top_at_mark_start == top, _next_marked_bytes == 0
   4.391 -        cur->add_to_marked_bytes(rspc.prev_marked_bytes());
   4.392 -        // _next_marked_bytes == prev_marked_bytes.
   4.393 -        cur->note_end_of_marking();
   4.394 -        // _prev_top_at_mark_start == top(),
   4.395 -        // _prev_marked_bytes == prev_marked_bytes
   4.396 -      }
   4.397 -      // If there is no mark in progress, we modified the _next variables
   4.398 -      // above needlessly, but harmlessly.
   4.399 -      if (_g1h->mark_in_progress()) {
   4.400 -        cur->note_start_of_marking(false);
   4.401 -        // _next_top_at_mark_start == top, _next_marked_bytes == 0
   4.402 -        // _next_marked_bytes == next_marked_bytes.
   4.403 -      }
   4.404 -    }
   4.405 -    cur = cur->next_in_collection_set();
   4.406 -  }
   4.407 +    rsfp_task.work(0);
   4.408 +  }
   4.409 +
   4.410 +  assert(check_cset_heap_region_claim_values(HeapRegion::ParEvacFailureClaimValue), "sanity");
   4.411 +
   4.412 +  // Reset the claim values in the regions in the collection set.
   4.413 +  reset_cset_heap_region_claim_values();
   4.414 +
   4.415 +  assert(check_cset_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity");
   4.416    assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!");
   4.417  
   4.418    // Now restore saved marks, if any.
   4.419 @@ -4148,6 +4030,7 @@
   4.420        markOop m = _preserved_marks_of_objs->at(i);
   4.421        obj->set_mark(m);
   4.422      }
   4.423 +
   4.424      // Delete the preserved marks growable arrays (allocated on the C heap).
   4.425      delete _objs_with_preserved_marks;
   4.426      delete _preserved_marks_of_objs;
   4.427 @@ -4172,8 +4055,7 @@
   4.428  
   4.429  oop
   4.430  G1CollectedHeap::handle_evacuation_failure_par(OopsInHeapRegionClosure* cl,
   4.431 -                                               oop old,
   4.432 -                                               bool should_mark_root) {
   4.433 +                                               oop old) {
   4.434    assert(obj_in_cs(old),
   4.435           err_msg("obj: "PTR_FORMAT" should still be in the CSet",
   4.436                   (HeapWord*) old));
   4.437 @@ -4182,15 +4064,6 @@
   4.438    if (forward_ptr == NULL) {
   4.439      // Forward-to-self succeeded.
   4.440  
   4.441 -    // should_mark_root will be true when this routine is called
   4.442 -    // from a root scanning closure during an initial mark pause.
   4.443 -    // In this case the thread that succeeds in self-forwarding the
   4.444 -    // object is also responsible for marking the object.
   4.445 -    if (should_mark_root) {
   4.446 -      assert(!oopDesc::is_null(old), "shouldn't be");
   4.447 -      _cm->grayRoot(old);
   4.448 -    }
   4.449 -
   4.450      if (_evac_failure_closure != cl) {
   4.451        MutexLockerEx x(EvacFailureStack_lock, Mutex::_no_safepoint_check_flag);
   4.452        assert(!_drain_in_progress,
   4.453 @@ -4286,30 +4159,8 @@
   4.454    return NULL;
   4.455  }
   4.456  
   4.457 -#ifndef PRODUCT
   4.458 -bool GCLabBitMapClosure::do_bit(size_t offset) {
   4.459 -  HeapWord* addr = _bitmap->offsetToHeapWord(offset);
   4.460 -  guarantee(_cm->isMarked(oop(addr)), "it should be!");
   4.461 -  return true;
   4.462 -}
   4.463 -#endif // PRODUCT
   4.464 -
   4.465  G1ParGCAllocBuffer::G1ParGCAllocBuffer(size_t gclab_word_size) :
   4.466 -  ParGCAllocBuffer(gclab_word_size),
   4.467 -  _should_mark_objects(false),
   4.468 -  _bitmap(G1CollectedHeap::heap()->reserved_region().start(), gclab_word_size),
   4.469 -  _retired(false)
   4.470 -{
   4.471 -  //_should_mark_objects is set to true when G1ParCopyHelper needs to
   4.472 -  // mark the forwarded location of an evacuated object.
   4.473 -  // We set _should_mark_objects to true if marking is active, i.e. when we
   4.474 -  // need to propagate a mark, or during an initial mark pause, i.e. when we
   4.475 -  // need to mark objects immediately reachable by the roots.
   4.476 -  if (G1CollectedHeap::heap()->mark_in_progress() ||
   4.477 -      G1CollectedHeap::heap()->g1_policy()->during_initial_mark_pause()) {
   4.478 -    _should_mark_objects = true;
   4.479 -  }
   4.480 -}
   4.481 +  ParGCAllocBuffer(gclab_word_size), _retired(false) { }
   4.482  
   4.483  G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
   4.484    : _g1h(g1h),
   4.485 @@ -4323,8 +4174,7 @@
   4.486      _tenured_alloc_buffer(g1h->desired_plab_sz(GCAllocForTenured)),
   4.487      _age_table(false),
   4.488      _strong_roots_time(0), _term_time(0),
   4.489 -    _alloc_buffer_waste(0), _undo_waste(0)
   4.490 -{
   4.491 +    _alloc_buffer_waste(0), _undo_waste(0) {
   4.492    // we allocate G1YoungSurvRateNumRegions plus one entries, since
   4.493    // we "sacrifice" entry 0 to keep track of surviving bytes for
   4.494    // non-young regions (where the age is -1)
   4.495 @@ -4429,35 +4279,53 @@
   4.496    } while (!refs()->is_empty());
   4.497  }
   4.498  
   4.499 -G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
   4.500 +G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1,
   4.501 +                                     G1ParScanThreadState* par_scan_state) :
   4.502    _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
   4.503    _par_scan_state(par_scan_state),
   4.504    _during_initial_mark(_g1->g1_policy()->during_initial_mark_pause()),
   4.505    _mark_in_progress(_g1->mark_in_progress()) { }
   4.506  
   4.507 -template <class T> void G1ParCopyHelper::mark_object(T* p) {
   4.508 -  // This is called from do_oop_work for objects that are not
   4.509 -  // in the collection set. Objects in the collection set
   4.510 -  // are marked after they have been evacuated.
   4.511 -
   4.512 -  T heap_oop = oopDesc::load_heap_oop(p);
   4.513 -  if (!oopDesc::is_null(heap_oop)) {
   4.514 -    oop obj = oopDesc::decode_heap_oop(heap_oop);
   4.515 -    HeapWord* addr = (HeapWord*)obj;
   4.516 -    if (_g1->is_in_g1_reserved(addr)) {
   4.517 -      _cm->grayRoot(oop(addr));
   4.518 -    }
   4.519 -  }
   4.520 -}
   4.521 -
   4.522 -oop G1ParCopyHelper::copy_to_survivor_space(oop old, bool should_mark_root,
   4.523 -                                                     bool should_mark_copy) {
   4.524 +void G1ParCopyHelper::mark_object(oop obj) {
   4.525 +#ifdef ASSERT
   4.526 +  HeapRegion* hr = _g1->heap_region_containing(obj);
   4.527 +  assert(hr != NULL, "sanity");
   4.528 +  assert(!hr->in_collection_set(), "should not mark objects in the CSet");
   4.529 +#endif // ASSERT
   4.530 +
   4.531 +  // We know that the object is not moving so it's safe to read its size.
   4.532 +  _cm->grayRoot(obj, (size_t) obj->size());
   4.533 +}
   4.534 +
   4.535 +void G1ParCopyHelper::mark_forwarded_object(oop from_obj, oop to_obj) {
   4.536 +#ifdef ASSERT
   4.537 +  assert(from_obj->is_forwarded(), "from obj should be forwarded");
   4.538 +  assert(from_obj->forwardee() == to_obj, "to obj should be the forwardee");
   4.539 +  assert(from_obj != to_obj, "should not be self-forwarded");
   4.540 +
   4.541 +  HeapRegion* from_hr = _g1->heap_region_containing(from_obj);
   4.542 +  assert(from_hr != NULL, "sanity");
   4.543 +  assert(from_hr->in_collection_set(), "from obj should be in the CSet");
   4.544 +
   4.545 +  HeapRegion* to_hr = _g1->heap_region_containing(to_obj);
   4.546 +  assert(to_hr != NULL, "sanity");
   4.547 +  assert(!to_hr->in_collection_set(), "should not mark objects in the CSet");
   4.548 +#endif // ASSERT
   4.549 +
   4.550 +  // The object might be in the process of being copied by another
   4.551 +  // worker so we cannot trust that its to-space image is
   4.552 +  // well-formed. So we have to read its size from its from-space
   4.553 +  // image which we know should not be changing.
   4.554 +  _cm->grayRoot(to_obj, (size_t) from_obj->size());
   4.555 +}
   4.556 +
   4.557 +oop G1ParCopyHelper::copy_to_survivor_space(oop old) {
   4.558    size_t    word_sz = old->size();
   4.559    HeapRegion* from_region = _g1->heap_region_containing_raw(old);
   4.560    // +1 to make the -1 indexes valid...
   4.561    int       young_index = from_region->young_index_in_cset()+1;
   4.562 -  assert( (from_region->is_young() && young_index > 0) ||
   4.563 -          (!from_region->is_young() && young_index == 0), "invariant" );
   4.564 +  assert( (from_region->is_young() && young_index >  0) ||
   4.565 +         (!from_region->is_young() && young_index == 0), "invariant" );
   4.566    G1CollectorPolicy* g1p = _g1->g1_policy();
   4.567    markOop m = old->mark();
   4.568    int age = m->has_displaced_mark_helper() ? m->displaced_mark_helper()->age()
   4.569 @@ -4471,7 +4339,7 @@
   4.570      // This will either forward-to-self, or detect that someone else has
   4.571      // installed a forwarding pointer.
   4.572      OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure();
   4.573 -    return _g1->handle_evacuation_failure_par(cl, old, should_mark_root);
   4.574 +    return _g1->handle_evacuation_failure_par(cl, old);
   4.575    }
   4.576  
   4.577    // We're going to allocate linearly, so might as well prefetch ahead.
   4.578 @@ -4507,28 +4375,14 @@
   4.579        obj->set_mark(m);
   4.580      }
   4.581  
   4.582 -    // Mark the evacuated object or propagate "next" mark bit
   4.583 -    if (should_mark_copy) {
   4.584 -      if (!use_local_bitmaps ||
   4.585 -          !_par_scan_state->alloc_buffer(alloc_purpose)->mark(obj_ptr)) {
   4.586 -        // if we couldn't mark it on the local bitmap (this happens when
   4.587 -        // the object was not allocated in the GCLab), we have to bite
   4.588 -        // the bullet and do the standard parallel mark
   4.589 -        _cm->markAndGrayObjectIfNecessary(obj);
   4.590 -      }
   4.591 -
   4.592 -      if (_g1->isMarkedNext(old)) {
   4.593 -        // Unmark the object's old location so that marking
   4.594 -        // doesn't think the old object is alive.
   4.595 -        _cm->nextMarkBitMap()->parClear((HeapWord*)old);
   4.596 -      }
   4.597 -    }
   4.598 -
   4.599      size_t* surv_young_words = _par_scan_state->surviving_young_words();
   4.600      surv_young_words[young_index] += word_sz;
   4.601  
   4.602      if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
   4.603 -      arrayOop(old)->set_length(0);
   4.604 +      // We keep track of the next start index in the length field of
   4.605 +      // the to-space object. The actual length can be found in the
   4.606 +      // length field of the from-space object.
   4.607 +      arrayOop(obj)->set_length(0);
   4.608        oop* old_p = set_partial_array_mask(old);
   4.609        _par_scan_state->push_on_queue(old_p);
   4.610      } else {
   4.611 @@ -4550,61 +4404,24 @@
   4.612  ::do_oop_work(T* p) {
   4.613    oop obj = oopDesc::load_decode_heap_oop(p);
   4.614    assert(barrier != G1BarrierRS || obj != NULL,
   4.615 -         "Precondition: G1BarrierRS implies obj is nonNull");
   4.616 -
   4.617 -  // Marking:
   4.618 -  // If the object is in the collection set, then the thread
   4.619 -  // that copies the object should mark, or propagate the
   4.620 -  // mark to, the evacuated object.
   4.621 -  // If the object is not in the collection set then we
   4.622 -  // should call the mark_object() method depending on the
   4.623 -  // value of the template parameter do_mark_object (which will
   4.624 -  // be true for root scanning closures during an initial mark
   4.625 -  // pause).
   4.626 -  // The mark_object() method first checks whether the object
   4.627 -  // is marked and, if not, attempts to mark the object.
   4.628 +         "Precondition: G1BarrierRS implies obj is non-NULL");
   4.629  
   4.630    // here the null check is implicit in the cset_fast_test() test
   4.631    if (_g1->in_cset_fast_test(obj)) {
   4.632 +    oop forwardee;
   4.633      if (obj->is_forwarded()) {
   4.634 -      oopDesc::encode_store_heap_oop(p, obj->forwardee());
   4.635 -      // If we are a root scanning closure during an initial
   4.636 -      // mark pause (i.e. do_mark_object will be true) then
   4.637 -      // we also need to handle marking of roots in the
   4.638 -      // event of an evacuation failure. In the event of an
   4.639 -      // evacuation failure, the object is forwarded to itself
   4.640 -      // and not copied. For root-scanning closures, the
   4.641 -      // object would be marked after a successful self-forward
   4.642 -      // but an object could be pointed to by both a root and non
   4.643 -      // root location and be self-forwarded by a non-root-scanning
   4.644 -      // closure. Therefore we also have to attempt to mark the
   4.645 -      // self-forwarded root object here.
   4.646 -      if (do_mark_object && obj->forwardee() == obj) {
   4.647 -        mark_object(p);
   4.648 -      }
   4.649 +      forwardee = obj->forwardee();
   4.650      } else {
   4.651 -      // During an initial mark pause, objects that are pointed to
   4.652 -      // by the roots need to be marked - even in the event of an
   4.653 -      // evacuation failure. We pass the template parameter
   4.654 -      // do_mark_object (which is true for root scanning closures
   4.655 -      // during an initial mark pause) to copy_to_survivor_space
   4.656 -      // which will pass it on to the evacuation failure handling
   4.657 -      // code. The thread that successfully self-forwards a root
   4.658 -      // object to itself is responsible for marking the object.
   4.659 -      bool should_mark_root = do_mark_object;
   4.660 -
   4.661 -      // We need to mark the copied object if we're a root scanning
   4.662 -      // closure during an initial mark pause (i.e. do_mark_object
   4.663 -      // will be true), or the object is already marked and we need
   4.664 -      // to propagate the mark to the evacuated copy.
   4.665 -      bool should_mark_copy = do_mark_object ||
   4.666 -                              _during_initial_mark ||
   4.667 -                              (_mark_in_progress && !_g1->is_obj_ill(obj));
   4.668 -
   4.669 -      oop copy_oop = copy_to_survivor_space(obj, should_mark_root,
   4.670 -                                                 should_mark_copy);
   4.671 -      oopDesc::encode_store_heap_oop(p, copy_oop);
   4.672 +      forwardee = copy_to_survivor_space(obj);
   4.673      }
   4.674 +    assert(forwardee != NULL, "forwardee should not be NULL");
   4.675 +    oopDesc::encode_store_heap_oop(p, forwardee);
   4.676 +    if (do_mark_object && forwardee != obj) {
   4.677 +      // If the object is self-forwarded we don't need to explicitly
   4.678 +      // mark it, the evacuation failure protocol will do so.
   4.679 +      mark_forwarded_object(obj, forwardee);
   4.680 +    }
   4.681 +
   4.682      // When scanning the RS, we only care about objs in CS.
   4.683      if (barrier == G1BarrierRS) {
   4.684        _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num());
   4.685 @@ -4613,8 +4430,8 @@
   4.686      // The object is not in collection set. If we're a root scanning
   4.687      // closure during an initial mark pause (i.e. do_mark_object will
   4.688      // be true) then attempt to mark the object.
   4.689 -    if (do_mark_object) {
   4.690 -      mark_object(p);
   4.691 +    if (do_mark_object && _g1->is_in_g1_reserved(obj)) {
   4.692 +      mark_object(obj);
   4.693      }
   4.694    }
   4.695  
   4.696 @@ -4632,35 +4449,51 @@
   4.697  
   4.698  template <class T> void G1ParScanPartialArrayClosure::do_oop_nv(T* p) {
   4.699    assert(has_partial_array_mask(p), "invariant");
   4.700 -  oop old = clear_partial_array_mask(p);
   4.701 -  assert(old->is_objArray(), "must be obj array");
   4.702 -  assert(old->is_forwarded(), "must be forwarded");
   4.703 -  assert(Universe::heap()->is_in_reserved(old), "must be in heap.");
   4.704 -
   4.705 -  objArrayOop obj = objArrayOop(old->forwardee());
   4.706 -  assert((void*)old != (void*)old->forwardee(), "self forwarding here?");
   4.707 -  // Process ParGCArrayScanChunk elements now
   4.708 -  // and push the remainder back onto queue
   4.709 -  int start     = arrayOop(old)->length();
   4.710 -  int end       = obj->length();
   4.711 -  int remainder = end - start;
   4.712 -  assert(start <= end, "just checking");
   4.713 +  oop from_obj = clear_partial_array_mask(p);
   4.714 +
   4.715 +  assert(Universe::heap()->is_in_reserved(from_obj), "must be in heap.");
   4.716 +  assert(from_obj->is_objArray(), "must be obj array");
   4.717 +  objArrayOop from_obj_array = objArrayOop(from_obj);
   4.718 +  // The from-space object contains the real length.
   4.719 +  int length                 = from_obj_array->length();
   4.720 +
   4.721 +  assert(from_obj->is_forwarded(), "must be forwarded");
   4.722 +  oop to_obj                 = from_obj->forwardee();
   4.723 +  assert(from_obj != to_obj, "should not be chunking self-forwarded objects");
   4.724 +  objArrayOop to_obj_array   = objArrayOop(to_obj);
   4.725 +  // We keep track of the next start index in the length field of the
   4.726 +  // to-space object.
   4.727 +  int next_index             = to_obj_array->length();
   4.728 +  assert(0 <= next_index && next_index < length,
   4.729 +         err_msg("invariant, next index: %d, length: %d", next_index, length));
   4.730 +
   4.731 +  int start                  = next_index;
   4.732 +  int end                    = length;
   4.733 +  int remainder              = end - start;
   4.734 +  // We'll try not to push a range that's smaller than ParGCArrayScanChunk.
   4.735    if (remainder > 2 * ParGCArrayScanChunk) {
   4.736 -    // Test above combines last partial chunk with a full chunk
   4.737      end = start + ParGCArrayScanChunk;
   4.738 -    arrayOop(old)->set_length(end);
   4.739 -    // Push remainder.
   4.740 -    oop* old_p = set_partial_array_mask(old);
   4.741 -    assert(arrayOop(old)->length() < obj->length(), "Empty push?");
   4.742 -    _par_scan_state->push_on_queue(old_p);
   4.743 +    to_obj_array->set_length(end);
   4.744 +    // Push the remainder before we process the range in case another
   4.745 +    // worker has run out of things to do and can steal it.
   4.746 +    oop* from_obj_p = set_partial_array_mask(from_obj);
   4.747 +    _par_scan_state->push_on_queue(from_obj_p);
   4.748    } else {
   4.749 -    // Restore length so that the heap remains parsable in
   4.750 -    // case of evacuation failure.
   4.751 -    arrayOop(old)->set_length(end);
   4.752 -  }
   4.753 -  _scanner.set_region(_g1->heap_region_containing_raw(obj));
   4.754 -  // process our set of indices (include header in first chunk)
   4.755 -  obj->oop_iterate_range(&_scanner, start, end);
   4.756 +    assert(length == end, "sanity");
   4.757 +    // We'll process the final range for this object. Restore the length
   4.758 +    // so that the heap remains parsable in case of evacuation failure.
   4.759 +    to_obj_array->set_length(end);
   4.760 +  }
   4.761 +  _scanner.set_region(_g1->heap_region_containing_raw(to_obj));
   4.762 +  // Process indexes [start,end). It will also process the header
   4.763 +  // along with the first chunk (i.e., the chunk with start == 0).
   4.764 +  // Note that at this point the length field of to_obj_array is not
   4.765 +  // correct given that we are using it to keep track of the next
   4.766 +  // start index. oop_iterate_range() (thankfully!) ignores the length
   4.767 +  // field and only relies on the start / end parameters.  It does
   4.768 +  // however return the size of the object which will be incorrect. So
   4.769 +  // we have to ignore it even if we wanted to use it.
   4.770 +  to_obj_array->oop_iterate_range(&_scanner, start, end);
   4.771  }
   4.772  
   4.773  class G1ParEvacuateFollowersClosure : public VoidClosure {
   4.774 @@ -4893,12 +4726,16 @@
   4.775  
   4.776    g1_policy()->record_ext_root_scan_time(worker_i, ext_root_time_ms);
   4.777  
   4.778 -  // Scan strong roots in mark stack.
   4.779 -  if (!_process_strong_tasks->is_task_claimed(G1H_PS_mark_stack_oops_do)) {
   4.780 -    concurrent_mark()->oops_do(scan_non_heap_roots);
   4.781 -  }
   4.782 -  double mark_stack_scan_ms = (os::elapsedTime() - ext_roots_end) * 1000.0;
   4.783 -  g1_policy()->record_mark_stack_scan_time(worker_i, mark_stack_scan_ms);
   4.784 +  // During conc marking we have to filter the per-thread SATB buffers
   4.785 +  // to make sure we remove any oops into the CSet (which will show up
   4.786 +  // as implicitly live).
   4.787 +  if (!_process_strong_tasks->is_task_claimed(G1H_PS_filter_satb_buffers)) {
   4.788 +    if (mark_in_progress()) {
   4.789 +      JavaThread::satb_mark_queue_set().filter_thread_buffers();
   4.790 +    }
   4.791 +  }
   4.792 +  double satb_filtering_ms = (os::elapsedTime() - ext_roots_end) * 1000.0;
   4.793 +  g1_policy()->record_satb_filtering_time(worker_i, satb_filtering_ms);
   4.794  
   4.795    // Now scan the complement of the collection set.
   4.796    if (scan_rs != NULL) {
   4.797 @@ -5439,6 +5276,7 @@
   4.798  }
   4.799  
   4.800  void G1CollectedHeap::evacuate_collection_set() {
   4.801 +  _expand_heap_after_alloc_failure = true;
   4.802    set_evacuation_failed(false);
   4.803  
   4.804    g1_rem_set()->prepare_for_oops_into_collection_set_do();
   4.805 @@ -5516,13 +5354,6 @@
   4.806  
   4.807    finalize_for_evac_failure();
   4.808  
   4.809 -  // Must do this before clearing the per-region evac-failure flags
   4.810 -  // (which is currently done when we free the collection set).
   4.811 -  // We also only do this if marking is actually in progress and so
   4.812 -  // have to do this before we set the mark_in_progress flag at the
   4.813 -  // end of an initial mark pause.
   4.814 -  concurrent_mark()->complete_marking_in_collection_set();
   4.815 -
   4.816    if (evacuation_failed()) {
   4.817      remove_self_forwarding_pointers();
   4.818      if (PrintGCDetails) {
   4.819 @@ -6179,6 +6010,8 @@
   4.820        } else {
   4.821          _hr_printer.alloc(new_alloc_region, G1HRPrinter::Old);
   4.822        }
   4.823 +      bool during_im = g1_policy()->during_initial_mark_pause();
   4.824 +      new_alloc_region->note_start_of_copying(during_im);
   4.825        return new_alloc_region;
   4.826      } else {
   4.827        g1_policy()->note_alloc_region_limit_reached(ap);
   4.828 @@ -6190,7 +6023,8 @@
   4.829  void G1CollectedHeap::retire_gc_alloc_region(HeapRegion* alloc_region,
   4.830                                               size_t allocated_bytes,
   4.831                                               GCAllocPurpose ap) {
   4.832 -  alloc_region->note_end_of_copying();
   4.833 +  bool during_im = g1_policy()->during_initial_mark_pause();
   4.834 +  alloc_region->note_end_of_copying(during_im);
   4.835    g1_policy()->record_bytes_copied_during_gc(allocated_bytes);
   4.836    if (ap == GCAllocForSurvived) {
   4.837      young_list()->add_survivor_region(alloc_region);

     5.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Thu Jan 12 09:59:55 2012 -0500
     5.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Fri Jan 13 01:55:22 2012 -0800
     5.3 @@ -1,5 +1,5 @@
     5.4  /*
     5.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
     5.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
     5.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     5.8   *
     5.9   * This code is free software; you can redistribute it and/or modify it
    5.10 @@ -285,6 +285,14 @@
    5.11    // Typically, it is not full so we should re-use it during the next GC.
    5.12    HeapRegion* _retained_old_gc_alloc_region;
    5.13  
    5.14 +  // It specifies whether we should attempt to expand the heap after a
    5.15 +  // region allocation failure. If heap expansion fails we set this to
    5.16 +  // false so that we don't re-attempt the heap expansion (it's likely
    5.17 +  // that subsequent expansion attempts will also fail if one fails).
    5.18 +  // Currently, it is only consulted during GC and it's reset at the
    5.19 +  // start of each GC.
    5.20 +  bool _expand_heap_after_alloc_failure;
    5.21 +
    5.22    // It resets the mutator alloc region before new allocations can take place.
    5.23    void init_mutator_alloc_region();
    5.24  
    5.25 @@ -861,8 +869,7 @@
    5.26    void finalize_for_evac_failure();
    5.27  
    5.28    // An attempt to evacuate "obj" has failed; take necessary steps.
    5.29 -  oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj,
    5.30 -                                    bool should_mark_root);
    5.31 +  oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj);
    5.32    void handle_evacuation_failure_common(oop obj, markOop m);
    5.33  
    5.34    // ("Weak") Reference processing support.
    5.35 @@ -954,7 +961,7 @@
    5.36    unsigned int* _worker_cset_start_region_time_stamp;
    5.37  
    5.38    enum G1H_process_strong_roots_tasks {
    5.39 -    G1H_PS_mark_stack_oops_do,
    5.40 +    G1H_PS_filter_satb_buffers,
    5.41      G1H_PS_refProcessor_oops_do,
    5.42      // Leave this one last.
    5.43      G1H_PS_NumElements
    5.44 @@ -1305,6 +1312,10 @@
    5.45    // It resets all the region claim values to the default.
    5.46    void reset_heap_region_claim_values();
    5.47  
    5.48 +  // Resets the claim values of regions in the current
    5.49 +  // collection set to the default.
    5.50 +  void reset_cset_heap_region_claim_values();
    5.51 +
    5.52  #ifdef ASSERT
    5.53    bool check_heap_region_claim_values(jint claim_value);
    5.54  
    5.55 @@ -1740,10 +1751,8 @@
    5.56        _gclab_word_size(gclab_word_size),
    5.57        _real_start_word(NULL),
    5.58        _real_end_word(NULL),
    5.59 -      _start_word(NULL)
    5.60 -  {
    5.61 -    guarantee( size_in_words() >= bitmap_size_in_words(),
    5.62 -               "just making sure");
    5.63 +      _start_word(NULL) {
    5.64 +    guarantee(false, "GCLabBitMap::GCLabBitmap(): don't call this any more");
    5.65    }
    5.66  
    5.67    inline unsigned heapWordToOffset(HeapWord* addr) {
    5.68 @@ -1797,6 +1806,8 @@
    5.69    }
    5.70  
    5.71    void set_buffer(HeapWord* start) {
    5.72 +    guarantee(false, "set_buffer(): don't call this any more");
    5.73 +
    5.74      guarantee(use_local_bitmaps, "invariant");
    5.75      clear();
    5.76  
    5.77 @@ -1820,6 +1831,8 @@
    5.78  #endif // PRODUCT
    5.79  
    5.80    void retire() {
    5.81 +    guarantee(false, "retire(): don't call this any more");
    5.82 +
    5.83      guarantee(use_local_bitmaps, "invariant");
    5.84      assert(fields_well_formed(), "invariant");
    5.85  
    5.86 @@ -1853,32 +1866,18 @@
    5.87  class G1ParGCAllocBuffer: public ParGCAllocBuffer {
    5.88  private:
    5.89    bool        _retired;
    5.90 -  bool        _should_mark_objects;
    5.91 -  GCLabBitMap _bitmap;
    5.92  
    5.93  public:
    5.94    G1ParGCAllocBuffer(size_t gclab_word_size);
    5.95  
    5.96 -  inline bool mark(HeapWord* addr) {
    5.97 -    guarantee(use_local_bitmaps, "invariant");
    5.98 -    assert(_should_mark_objects, "invariant");
    5.99 -    return _bitmap.mark(addr);
   5.100 -  }
   5.101 -
   5.102 -  inline void set_buf(HeapWord* buf) {
   5.103 -    if (use_local_bitmaps && _should_mark_objects) {
   5.104 -      _bitmap.set_buffer(buf);
   5.105 -    }
   5.106 +  void set_buf(HeapWord* buf) {
   5.107      ParGCAllocBuffer::set_buf(buf);
   5.108      _retired = false;
   5.109    }
   5.110  
   5.111 -  inline void retire(bool end_of_gc, bool retain) {
   5.112 +  void retire(bool end_of_gc, bool retain) {
   5.113      if (_retired)
   5.114        return;
   5.115 -    if (use_local_bitmaps && _should_mark_objects) {
   5.116 -      _bitmap.retire();
   5.117 -    }
   5.118      ParGCAllocBuffer::retire(end_of_gc, retain);
   5.119      _retired = true;
   5.120    }

     6.1 --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp	Thu Jan 12 09:59:55 2012 -0500
     6.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp	Fri Jan 13 01:55:22 2012 -0800
     6.3 @@ -1,5 +1,5 @@
     6.4  /*
     6.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
     6.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
     6.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     6.8   *
     6.9   * This code is free software; you can redistribute it and/or modify it
    6.10 @@ -281,7 +281,7 @@
    6.11  
    6.12    _par_last_gc_worker_start_times_ms = new double[_parallel_gc_threads];
    6.13    _par_last_ext_root_scan_times_ms = new double[_parallel_gc_threads];
    6.14 -  _par_last_mark_stack_scan_times_ms = new double[_parallel_gc_threads];
    6.15 +  _par_last_satb_filtering_times_ms = new double[_parallel_gc_threads];
    6.16  
    6.17    _par_last_update_rs_times_ms = new double[_parallel_gc_threads];
    6.18    _par_last_update_rs_processed_buffers = new double[_parallel_gc_threads];
    6.19 @@ -905,10 +905,19 @@
    6.20      gclog_or_tty->print(" (%s)", gcs_are_young() ? "young" : "mixed");
    6.21    }
    6.22  
    6.23 -  // We only need to do this here as the policy will only be applied
    6.24 -  // to the GC we're about to start. so, no point is calculating this
    6.25 -  // every time we calculate / recalculate the target young length.
    6.26 -  update_survivors_policy();
    6.27 +  if (!during_initial_mark_pause()) {
    6.28 +    // We only need to do this here as the policy will only be applied
    6.29 +    // to the GC we're about to start. so, no point is calculating this
    6.30 +    // every time we calculate / recalculate the target young length.
    6.31 +    update_survivors_policy();
    6.32 +  } else {
    6.33 +    // The marking phase has a "we only copy implicitly live
    6.34 +    // objects during marking" invariant. The easiest way to ensure it
    6.35 +    // holds is not to allocate any survivor regions and tenure all
    6.36 +    // objects. In the future we might change this and handle survivor
    6.37 +    // regions specially during marking.
    6.38 +    tenure_all_objects();
    6.39 +  }
    6.40  
    6.41    assert(_g1->used() == _g1->recalculate_used(),
    6.42           err_msg("sanity, used: "SIZE_FORMAT" recalculate_used: "SIZE_FORMAT,
    6.43 @@ -939,7 +948,7 @@
    6.44    for (int i = 0; i < _parallel_gc_threads; ++i) {
    6.45      _par_last_gc_worker_start_times_ms[i] = -1234.0;
    6.46      _par_last_ext_root_scan_times_ms[i] = -1234.0;
    6.47 -    _par_last_mark_stack_scan_times_ms[i] = -1234.0;
    6.48 +    _par_last_satb_filtering_times_ms[i] = -1234.0;
    6.49      _par_last_update_rs_times_ms[i] = -1234.0;
    6.50      _par_last_update_rs_processed_buffers[i] = -1234.0;
    6.51      _par_last_scan_rs_times_ms[i] = -1234.0;
    6.52 @@ -1227,7 +1236,7 @@
    6.53    // of the PrintGCDetails output, in the non-parallel case.
    6.54  
    6.55    double ext_root_scan_time = avg_value(_par_last_ext_root_scan_times_ms);
    6.56 -  double mark_stack_scan_time = avg_value(_par_last_mark_stack_scan_times_ms);
    6.57 +  double satb_filtering_time = avg_value(_par_last_satb_filtering_times_ms);
    6.58    double update_rs_time = avg_value(_par_last_update_rs_times_ms);
    6.59    double update_rs_processed_buffers =
    6.60      sum_of_values(_par_last_update_rs_processed_buffers);
    6.61 @@ -1236,7 +1245,7 @@
    6.62    double termination_time = avg_value(_par_last_termination_times_ms);
    6.63  
    6.64    double known_time = ext_root_scan_time +
    6.65 -                      mark_stack_scan_time +
    6.66 +                      satb_filtering_time +
    6.67                        update_rs_time +
    6.68                        scan_rs_time +
    6.69                        obj_copy_time;
    6.70 @@ -1282,7 +1291,7 @@
    6.71      body_summary->record_satb_drain_time_ms(_cur_satb_drain_time_ms);
    6.72  
    6.73      body_summary->record_ext_root_scan_time_ms(ext_root_scan_time);
    6.74 -    body_summary->record_mark_stack_scan_time_ms(mark_stack_scan_time);
    6.75 +    body_summary->record_satb_filtering_time_ms(satb_filtering_time);
    6.76      body_summary->record_update_rs_time_ms(update_rs_time);
    6.77      body_summary->record_scan_rs_time_ms(scan_rs_time);
    6.78      body_summary->record_obj_copy_time_ms(obj_copy_time);
    6.79 @@ -1376,16 +1385,12 @@
    6.80                             (last_pause_included_initial_mark) ? " (initial-mark)" : "",
    6.81                             elapsed_ms / 1000.0);
    6.82  
    6.83 -    if (print_marking_info) {
    6.84 -      print_stats(1, "SATB Drain Time", _cur_satb_drain_time_ms);
    6.85 -    }
    6.86 -
    6.87      if (parallel) {
    6.88        print_stats(1, "Parallel Time", _cur_collection_par_time_ms);
    6.89        print_par_stats(2, "GC Worker Start", _par_last_gc_worker_start_times_ms);
    6.90        print_par_stats(2, "Ext Root Scanning", _par_last_ext_root_scan_times_ms);
    6.91        if (print_marking_info) {
    6.92 -        print_par_stats(2, "Mark Stack Scanning", _par_last_mark_stack_scan_times_ms);
    6.93 +        print_par_stats(2, "SATB Filtering", _par_last_satb_filtering_times_ms);
    6.94        }
    6.95        print_par_stats(2, "Update RS", _par_last_update_rs_times_ms);
    6.96        print_par_sizes(3, "Processed Buffers", _par_last_update_rs_processed_buffers);
    6.97 @@ -1399,7 +1404,7 @@
    6.98          _par_last_gc_worker_times_ms[i] = _par_last_gc_worker_end_times_ms[i] - _par_last_gc_worker_start_times_ms[i];
    6.99  
   6.100          double worker_known_time = _par_last_ext_root_scan_times_ms[i] +
   6.101 -                                   _par_last_mark_stack_scan_times_ms[i] +
   6.102 +                                   _par_last_satb_filtering_times_ms[i] +
   6.103                                     _par_last_update_rs_times_ms[i] +
   6.104                                     _par_last_scan_rs_times_ms[i] +
   6.105                                     _par_last_obj_copy_times_ms[i] +
   6.106 @@ -1412,7 +1417,7 @@
   6.107      } else {
   6.108        print_stats(1, "Ext Root Scanning", ext_root_scan_time);
   6.109        if (print_marking_info) {
   6.110 -        print_stats(1, "Mark Stack Scanning", mark_stack_scan_time);
   6.111 +        print_stats(1, "SATB Filtering", satb_filtering_time);
   6.112        }
   6.113        print_stats(1, "Update RS", update_rs_time);
   6.114        print_stats(2, "Processed Buffers", (int)update_rs_processed_buffers);
   6.115 @@ -1983,11 +1988,10 @@
   6.116    if (summary->get_total_seq()->num() > 0) {
   6.117      print_summary_sd(0, "Evacuation Pauses", summary->get_total_seq());
   6.118      if (body_summary != NULL) {
   6.119 -      print_summary(1, "SATB Drain", body_summary->get_satb_drain_seq());
   6.120        if (parallel) {
   6.121          print_summary(1, "Parallel Time", body_summary->get_parallel_seq());
   6.122          print_summary(2, "Ext Root Scanning", body_summary->get_ext_root_scan_seq());
   6.123 -        print_summary(2, "Mark Stack Scanning", body_summary->get_mark_stack_scan_seq());
   6.124 +        print_summary(2, "SATB Filtering", body_summary->get_satb_filtering_seq());
   6.125          print_summary(2, "Update RS", body_summary->get_update_rs_seq());
   6.126          print_summary(2, "Scan RS", body_summary->get_scan_rs_seq());
   6.127          print_summary(2, "Object Copy", body_summary->get_obj_copy_seq());
   6.128 @@ -1996,7 +2000,7 @@
   6.129          {
   6.130            NumberSeq* other_parts[] = {
   6.131              body_summary->get_ext_root_scan_seq(),
   6.132 -            body_summary->get_mark_stack_scan_seq(),
   6.133 +            body_summary->get_satb_filtering_seq(),
   6.134              body_summary->get_update_rs_seq(),
   6.135              body_summary->get_scan_rs_seq(),
   6.136              body_summary->get_obj_copy_seq(),
   6.137 @@ -2009,7 +2013,7 @@
   6.138          }
   6.139        } else {
   6.140          print_summary(1, "Ext Root Scanning", body_summary->get_ext_root_scan_seq());
   6.141 -        print_summary(1, "Mark Stack Scanning", body_summary->get_mark_stack_scan_seq());
   6.142 +        print_summary(1, "SATB Filtering", body_summary->get_satb_filtering_seq());
   6.143          print_summary(1, "Update RS", body_summary->get_update_rs_seq());
   6.144          print_summary(1, "Scan RS", body_summary->get_scan_rs_seq());
   6.145          print_summary(1, "Object Copy", body_summary->get_obj_copy_seq());
   6.146 @@ -2036,7 +2040,7 @@
   6.147              body_summary->get_satb_drain_seq(),
   6.148              body_summary->get_update_rs_seq(),
   6.149              body_summary->get_ext_root_scan_seq(),
   6.150 -            body_summary->get_mark_stack_scan_seq(),
   6.151 +            body_summary->get_satb_filtering_seq(),
   6.152              body_summary->get_scan_rs_seq(),
   6.153              body_summary->get_obj_copy_seq()
   6.154            };
   6.155 @@ -2433,9 +2437,6 @@
   6.156    assert(_inc_cset_build_state == Active, "Precondition");
   6.157    assert(!hr->is_young(), "non-incremental add of young region");
   6.158  
   6.159 -  if (_g1->mark_in_progress())
   6.160 -    _g1->concurrent_mark()->registerCSetRegion(hr);
   6.161 -
   6.162    assert(!hr->in_collection_set(), "should not already be in the CSet");
   6.163    hr->set_in_collection_set(true);
   6.164    hr->set_next_in_collection_set(_collection_set);
   6.165 @@ -2705,9 +2706,6 @@
   6.166    // Clear the fields that point to the survivor list - they are all young now.
   6.167    young_list->clear_survivors();
   6.168  
   6.169 -  if (_g1->mark_in_progress())
   6.170 -    _g1->concurrent_mark()->register_collection_set_finger(_inc_cset_max_finger);
   6.171 -
   6.172    _collection_set = _inc_cset_head;
   6.173    _collection_set_bytes_used_before = _inc_cset_bytes_used_before;
   6.174    time_remaining_ms -= _inc_cset_predicted_elapsed_time_ms;

     7.1 --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp	Thu Jan 12 09:59:55 2012 -0500
     7.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp	Fri Jan 13 01:55:22 2012 -0800
     7.3 @@ -1,5 +1,5 @@
     7.4  /*
     7.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
     7.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
     7.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     7.8   *
     7.9   * This code is free software; you can redistribute it and/or modify it
    7.10 @@ -67,7 +67,7 @@
    7.11    define_num_seq(satb_drain) // optional
    7.12    define_num_seq(parallel) // parallel only
    7.13      define_num_seq(ext_root_scan)
    7.14 -    define_num_seq(mark_stack_scan)
    7.15 +    define_num_seq(satb_filtering)
    7.16      define_num_seq(update_rs)
    7.17      define_num_seq(scan_rs)
    7.18      define_num_seq(obj_copy)
    7.19 @@ -215,7 +215,7 @@
    7.20  
    7.21    double* _par_last_gc_worker_start_times_ms;
    7.22    double* _par_last_ext_root_scan_times_ms;
    7.23 -  double* _par_last_mark_stack_scan_times_ms;
    7.24 +  double* _par_last_satb_filtering_times_ms;
    7.25    double* _par_last_update_rs_times_ms;
    7.26    double* _par_last_update_rs_processed_buffers;
    7.27    double* _par_last_scan_rs_times_ms;
    7.28 @@ -841,8 +841,8 @@
    7.29      _par_last_ext_root_scan_times_ms[worker_i] = ms;
    7.30    }
    7.31  
    7.32 -  void record_mark_stack_scan_time(int worker_i, double ms) {
    7.33 -    _par_last_mark_stack_scan_times_ms[worker_i] = ms;
    7.34 +  void record_satb_filtering_time(int worker_i, double ms) {
    7.35 +    _par_last_satb_filtering_times_ms[worker_i] = ms;
    7.36    }
    7.37  
    7.38    void record_satb_drain_time(double ms) {
    7.39 @@ -1146,6 +1146,11 @@
    7.40      _survivor_surv_rate_group->stop_adding_regions();
    7.41    }
    7.42  
    7.43 +  void tenure_all_objects() {
    7.44 +    _max_survivor_regions = 0;
    7.45 +    _tenuring_threshold = 0;
    7.46 +  }
    7.47 +
    7.48    void record_survivor_regions(size_t      regions,
    7.49                                 HeapRegion* head,
    7.50                                 HeapRegion* tail) {

     8.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     8.2 +++ b/src/share/vm/gc_implementation/g1/g1EvacFailure.hpp	Fri Jan 13 01:55:22 2012 -0800
     8.3 @@ -0,0 +1,236 @@
     8.4 +/*
     8.5 + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
     8.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     8.7 + *
     8.8 + * This code is free software; you can redistribute it and/or modify it
     8.9 + * under the terms of the GNU General Public License version 2 only, as
    8.10 + * published by the Free Software Foundation.
    8.11 + *
    8.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
    8.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    8.14 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    8.15 + * version 2 for more details (a copy is included in the LICENSE file that
    8.16 + * accompanied this code).
    8.17 + *
    8.18 + * You should have received a copy of the GNU General Public License version
    8.19 + * 2 along with this work; if not, write to the Free Software Foundation,
    8.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
    8.21 + *
    8.22 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
    8.23 + * or visit www.oracle.com if you need additional information or have any
    8.24 + * questions.
    8.25 + *
    8.26 + */
    8.27 +
    8.28 +#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP
    8.29 +#define SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP
    8.30 +
    8.31 +#include "gc_implementation/g1/concurrentMark.inline.hpp"
    8.32 +#include "gc_implementation/g1/dirtyCardQueue.hpp"
    8.33 +#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
    8.34 +#include "gc_implementation/g1/g1_globals.hpp"
    8.35 +#include "gc_implementation/g1/g1OopClosures.inline.hpp"
    8.36 +#include "gc_implementation/g1/heapRegion.hpp"
    8.37 +#include "gc_implementation/g1/heapRegionRemSet.hpp"
    8.38 +#include "utilities/workgroup.hpp"
    8.39 +
    8.40 +// Closures and tasks associated with any self-forwarding pointers
    8.41 +// installed as a result of an evacuation failure.
    8.42 +
    8.43 +class UpdateRSetDeferred : public OopsInHeapRegionClosure {
    8.44 +private:
    8.45 +  G1CollectedHeap* _g1;
    8.46 +  DirtyCardQueue *_dcq;
    8.47 +  CardTableModRefBS* _ct_bs;
    8.48 +
    8.49 +public:
    8.50 +  UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) :
    8.51 +    _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {}
    8.52 +
    8.53 +  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
    8.54 +  virtual void do_oop(      oop* p) { do_oop_work(p); }
    8.55 +  template <class T> void do_oop_work(T* p) {
    8.56 +    assert(_from->is_in_reserved(p), "paranoia");
    8.57 +    if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) &&
    8.58 +        !_from->is_survivor()) {
    8.59 +      size_t card_index = _ct_bs->index_for(p);
    8.60 +      if (_ct_bs->mark_card_deferred(card_index)) {
    8.61 +        _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index));
    8.62 +      }
    8.63 +    }
    8.64 +  }
    8.65 +};
    8.66 +
    8.67 +class RemoveSelfForwardPtrObjClosure: public ObjectClosure {
    8.68 +private:
    8.69 +  G1CollectedHeap* _g1;
    8.70 +  ConcurrentMark* _cm;
    8.71 +  HeapRegion* _hr;
    8.72 +  size_t _marked_bytes;
    8.73 +  OopsInHeapRegionClosure *_update_rset_cl;
    8.74 +  bool _during_initial_mark;
    8.75 +  bool _during_conc_mark;
    8.76 +public:
    8.77 +  RemoveSelfForwardPtrObjClosure(G1CollectedHeap* g1, ConcurrentMark* cm,
    8.78 +                                 HeapRegion* hr,
    8.79 +                                 OopsInHeapRegionClosure* update_rset_cl,
    8.80 +                                 bool during_initial_mark,
    8.81 +                                 bool during_conc_mark) :
    8.82 +    _g1(g1), _cm(cm), _hr(hr), _marked_bytes(0),
    8.83 +    _update_rset_cl(update_rset_cl),
    8.84 +    _during_initial_mark(during_initial_mark),
    8.85 +    _during_conc_mark(during_conc_mark) { }
    8.86 +
    8.87 +  size_t marked_bytes() { return _marked_bytes; }
    8.88 +
    8.89 +  // <original comment>
    8.90 +  // The original idea here was to coalesce evacuated and dead objects.
    8.91 +  // However that caused complications with the block offset table (BOT).
    8.92 +  // In particular if there were two TLABs, one of them partially refined.
    8.93 +  // |----- TLAB_1--------|----TLAB_2-~~~(partially refined part)~~~|
    8.94 +  // The BOT entries of the unrefined part of TLAB_2 point to the start
    8.95 +  // of TLAB_2. If the last object of the TLAB_1 and the first object
    8.96 +  // of TLAB_2 are coalesced, then the cards of the unrefined part
    8.97 +  // would point into middle of the filler object.
    8.98 +  // The current approach is to not coalesce and leave the BOT contents intact.
    8.99 +  // </original comment>
   8.100 +  //
   8.101 +  // We now reset the BOT when we start the object iteration over the
   8.102 +  // region and refine its entries for every object we come across. So
   8.103 +  // the above comment is not really relevant and we should be able
   8.104 +  // to coalesce dead objects if we want to.
   8.105 +  void do_object(oop obj) {
   8.106 +    HeapWord* obj_addr = (HeapWord*) obj;
   8.107 +    assert(_hr->is_in(obj_addr), "sanity");
   8.108 +    size_t obj_size = obj->size();
   8.109 +    _hr->update_bot_for_object(obj_addr, obj_size);
   8.110 +
   8.111 +    if (obj->is_forwarded() && obj->forwardee() == obj) {
   8.112 +      // The object failed to move.
   8.113 +
   8.114 +      // We consider all objects that we find self-forwarded to be
   8.115 +      // live. What we'll do is that we'll update the prev marking
   8.116 +      // info so that they are all under PTAMS and explicitly marked.
   8.117 +      _cm->markPrev(obj);
   8.118 +      if (_during_initial_mark) {
   8.119 +        // For the next marking info we'll only mark the
   8.120 +        // self-forwarded objects explicitly if we are during
   8.121 +        // initial-mark (since, normally, we only mark objects pointed
   8.122 +        // to by roots if we succeed in copying them). By marking all
   8.123 +        // self-forwarded objects we ensure that we mark any that are
   8.124 +        // still pointed to be roots. During concurrent marking, and
   8.125 +        // after initial-mark, we don't need to mark any objects
   8.126 +        // explicitly and all objects in the CSet are considered
   8.127 +        // (implicitly) live. So, we won't mark them explicitly and
   8.128 +        // we'll leave them over NTAMS.
   8.129 +        _cm->markNext(obj);
   8.130 +      }
   8.131 +      _marked_bytes += (obj_size * HeapWordSize);
   8.132 +      obj->set_mark(markOopDesc::prototype());
   8.133 +
   8.134 +      // While we were processing RSet buffers during the collection,
   8.135 +      // we actually didn't scan any cards on the collection set,
   8.136 +      // since we didn't want to update remembered sets with entries
   8.137 +      // that point into the collection set, given that live objects
   8.138 +      // from the collection set are about to move and such entries
   8.139 +      // will be stale very soon.
   8.140 +      // This change also dealt with a reliability issue which
   8.141 +      // involved scanning a card in the collection set and coming
   8.142 +      // across an array that was being chunked and looking malformed.
   8.143 +      // The problem is that, if evacuation fails, we might have
   8.144 +      // remembered set entries missing given that we skipped cards on
   8.145 +      // the collection set. So, we'll recreate such entries now.
   8.146 +      obj->oop_iterate(_update_rset_cl);
   8.147 +      assert(_cm->isPrevMarked(obj), "Should be marked!");
   8.148 +    } else {
   8.149 +      // The object has been either evacuated or is dead. Fill it with a
   8.150 +      // dummy object.
   8.151 +      MemRegion mr((HeapWord*) obj, obj_size);
   8.152 +      CollectedHeap::fill_with_object(mr);
   8.153 +    }
   8.154 +  }
   8.155 +};
   8.156 +
   8.157 +class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure {
   8.158 +  G1CollectedHeap* _g1h;
   8.159 +  ConcurrentMark* _cm;
   8.160 +  OopsInHeapRegionClosure *_update_rset_cl;
   8.161 +
   8.162 +public:
   8.163 +  RemoveSelfForwardPtrHRClosure(G1CollectedHeap* g1h,
   8.164 +                                OopsInHeapRegionClosure* update_rset_cl) :
   8.165 +    _g1h(g1h), _update_rset_cl(update_rset_cl),
   8.166 +    _cm(_g1h->concurrent_mark()) { }
   8.167 +
   8.168 +  bool doHeapRegion(HeapRegion *hr) {
   8.169 +    bool during_initial_mark = _g1h->g1_policy()->during_initial_mark_pause();
   8.170 +    bool during_conc_mark = _g1h->mark_in_progress();
   8.171 +
   8.172 +    assert(!hr->isHumongous(), "sanity");
   8.173 +    assert(hr->in_collection_set(), "bad CS");
   8.174 +
   8.175 +    if (hr->claimHeapRegion(HeapRegion::ParEvacFailureClaimValue)) {
   8.176 +      if (hr->evacuation_failed()) {
   8.177 +        RemoveSelfForwardPtrObjClosure rspc(_g1h, _cm, hr, _update_rset_cl,
   8.178 +                                            during_initial_mark,
   8.179 +                                            during_conc_mark);
   8.180 +
   8.181 +        MemRegion mr(hr->bottom(), hr->end());
   8.182 +        // We'll recreate the prev marking info so we'll first clear
   8.183 +        // the prev bitmap range for this region. We never mark any
   8.184 +        // CSet objects explicitly so the next bitmap range should be
   8.185 +        // cleared anyway.
   8.186 +        _cm->clearRangePrevBitmap(mr);
   8.187 +
   8.188 +        hr->note_self_forwarding_removal_start(during_initial_mark,
   8.189 +                                               during_conc_mark);
   8.190 +
   8.191 +        // In the common case (i.e. when there is no evacuation
   8.192 +        // failure) we make sure that the following is done when
   8.193 +        // the region is freed so that it is "ready-to-go" when it's
   8.194 +        // re-allocated. However, when evacuation failure happens, a
   8.195 +        // region will remain in the heap and might ultimately be added
   8.196 +        // to a CSet in the future. So we have to be careful here and
   8.197 +        // make sure the region's RSet is ready for parallel iteration
   8.198 +        // whenever this might be required in the future.
   8.199 +        hr->rem_set()->reset_for_par_iteration();
   8.200 +        hr->reset_bot();
   8.201 +        _update_rset_cl->set_region(hr);
   8.202 +        hr->object_iterate(&rspc);
   8.203 +
   8.204 +        hr->note_self_forwarding_removal_end(during_initial_mark,
   8.205 +                                             during_conc_mark,
   8.206 +                                             rspc.marked_bytes());
   8.207 +      }
   8.208 +    }
   8.209 +    return false;
   8.210 +  }
   8.211 +};
   8.212 +
   8.213 +class G1ParRemoveSelfForwardPtrsTask: public AbstractGangTask {
   8.214 +protected:
   8.215 +  G1CollectedHeap* _g1h;
   8.216 +
   8.217 +public:
   8.218 +  G1ParRemoveSelfForwardPtrsTask(G1CollectedHeap* g1h) :
   8.219 +    AbstractGangTask("G1 Remove Self-forwarding Pointers"),
   8.220 +    _g1h(g1h) { }
   8.221 +
   8.222 +  void work(uint worker_id) {
   8.223 +    UpdateRSetImmediate immediate_update(_g1h->g1_rem_set());
   8.224 +    DirtyCardQueue dcq(&_g1h->dirty_card_queue_set());
   8.225 +    UpdateRSetDeferred deferred_update(_g1h, &dcq);
   8.226 +
   8.227 +    OopsInHeapRegionClosure *update_rset_cl = &deferred_update;
   8.228 +    if (!G1DeferredRSUpdate) {
   8.229 +      update_rset_cl = &immediate_update;
   8.230 +    }
   8.231 +
   8.232 +    RemoveSelfForwardPtrHRClosure rsfp_cl(_g1h, update_rset_cl);
   8.233 +
   8.234 +    HeapRegion* hr = _g1h->start_cset_region_for_worker(worker_id);
   8.235 +    _g1h->collection_set_iterate_from(hr, &rsfp_cl);
   8.236 +  }
   8.237 +};
   8.238 +
   8.239 +#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP

     9.1 --- a/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	Thu Jan 12 09:59:55 2012 -0500
     9.2 +++ b/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	Fri Jan 13 01:55:22 2012 -0800
     9.3 @@ -1,5 +1,5 @@
     9.4  /*
     9.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
     9.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
     9.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     9.8   *
     9.9   * This code is free software; you can redistribute it and/or modify it
    9.10 @@ -121,17 +121,25 @@
    9.11  class G1ParCopyHelper : public G1ParClosureSuper {
    9.12    G1ParScanClosure *_scanner;
    9.13  protected:
    9.14 -  template <class T> void mark_object(T* p);
    9.15 -  oop copy_to_survivor_space(oop obj, bool should_mark_root,
    9.16 -                                      bool should_mark_copy);
    9.17 +  // Mark the object if it's not already marked. This is used to mark
    9.18 +  // objects pointed to by roots that are guaranteed not to move
    9.19 +  // during the GC (i.e., non-CSet objects). It is MT-safe.
    9.20 +  void mark_object(oop obj);
    9.21 +
    9.22 +  // Mark the object if it's not already marked. This is used to mark
    9.23 +  // objects pointed to by roots that have been forwarded during a
    9.24 +  // GC. It is MT-safe.
    9.25 +  void mark_forwarded_object(oop from_obj, oop to_obj);
    9.26 +
    9.27 +  oop copy_to_survivor_space(oop obj);
    9.28 +
    9.29  public:
    9.30    G1ParCopyHelper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state,
    9.31                    G1ParScanClosure *scanner) :
    9.32      G1ParClosureSuper(g1, par_scan_state), _scanner(scanner) { }
    9.33  };
    9.34  
    9.35 -template<bool do_gen_barrier, G1Barrier barrier,
    9.36 -         bool do_mark_object>
    9.37 +template <bool do_gen_barrier, G1Barrier barrier, bool do_mark_object>
    9.38  class G1ParCopyClosure : public G1ParCopyHelper {
    9.39    G1ParScanClosure _scanner;
    9.40  
    9.41 @@ -140,9 +148,8 @@
    9.42  public:
    9.43    G1ParCopyClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state,
    9.44                     ReferenceProcessor* rp) :
    9.45 -    _scanner(g1, par_scan_state, rp),
    9.46 -    G1ParCopyHelper(g1, par_scan_state, &_scanner)
    9.47 -  {
    9.48 +      _scanner(g1, par_scan_state, rp),
    9.49 +      G1ParCopyHelper(g1, par_scan_state, &_scanner) {
    9.50      assert(_ref_processor == NULL, "sanity");
    9.51    }
    9.52  

    10.1 --- a/src/share/vm/gc_implementation/g1/g1_globals.hpp	Thu Jan 12 09:59:55 2012 -0500
    10.2 +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp	Fri Jan 13 01:55:22 2012 -0800
    10.3 @@ -295,7 +295,7 @@
    10.4            "Percentage (0-100) of the heap size to use as minimum "          \
    10.5            "young gen size.")                                                \
    10.6                                                                              \
    10.7 -  develop(uintx, G1DefaultMaxNewGenPercent, 50,                             \
    10.8 +  develop(uintx, G1DefaultMaxNewGenPercent, 80,                             \
    10.9            "Percentage (0-100) of the heap size to use as maximum "          \
   10.10            "young gen size.")
   10.11  

    11.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.cpp	Thu Jan 12 09:59:55 2012 -0500
    11.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp	Fri Jan 13 01:55:22 2012 -0800
    11.3 @@ -1,5 +1,5 @@
    11.4  /*
    11.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
    11.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
    11.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    11.8   *
    11.9   * This code is free software; you can redistribute it and/or modify it
   11.10 @@ -575,6 +575,40 @@
   11.11    oops_in_mr_iterate(MemRegion(bottom(), saved_mark_word()), cl);
   11.12  }
   11.13  
   11.14 +void HeapRegion::note_self_forwarding_removal_start(bool during_initial_mark,
   11.15 +                                                    bool during_conc_mark) {
   11.16 +  // We always recreate the prev marking info and we'll explicitly
   11.17 +  // mark all objects we find to be self-forwarded on the prev
   11.18 +  // bitmap. So all objects need to be below PTAMS.
   11.19 +  _prev_top_at_mark_start = top();
   11.20 +  _prev_marked_bytes = 0;
   11.21 +
   11.22 +  if (during_initial_mark) {
   11.23 +    // During initial-mark, we'll also explicitly mark all objects
   11.24 +    // we find to be self-forwarded on the next bitmap. So all
   11.25 +    // objects need to be below NTAMS.
   11.26 +    _next_top_at_mark_start = top();
   11.27 +    set_top_at_conc_mark_count(bottom());
   11.28 +    _next_marked_bytes = 0;
   11.29 +  } else if (during_conc_mark) {
   11.30 +    // During concurrent mark, all objects in the CSet (including
   11.31 +    // the ones we find to be self-forwarded) are implicitly live.
   11.32 +    // So all objects need to be above NTAMS.
   11.33 +    _next_top_at_mark_start = bottom();
   11.34 +    set_top_at_conc_mark_count(bottom());
   11.35 +    _next_marked_bytes = 0;
   11.36 +  }
   11.37 +}
   11.38 +
   11.39 +void HeapRegion::note_self_forwarding_removal_end(bool during_initial_mark,
   11.40 +                                                  bool during_conc_mark,
   11.41 +                                                  size_t marked_bytes) {
   11.42 +  assert(0 <= marked_bytes && marked_bytes <= used(),
   11.43 +         err_msg("marked: "SIZE_FORMAT" used: "SIZE_FORMAT,
   11.44 +                 marked_bytes, used()));
   11.45 +  _prev_marked_bytes = marked_bytes;
   11.46 +}
   11.47 +
   11.48  HeapWord*
   11.49  HeapRegion::object_iterate_mem_careful(MemRegion mr,
   11.50                                                   ObjectClosure* cl) {

    12.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.hpp	Thu Jan 12 09:59:55 2012 -0500
    12.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp	Fri Jan 13 01:55:22 2012 -0800
    12.3 @@ -1,5 +1,5 @@
    12.4  /*
    12.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
    12.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
    12.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    12.8   *
    12.9   * This code is free software; you can redistribute it and/or modify it
   12.10 @@ -373,7 +373,8 @@
   12.11      ScrubRemSetClaimValue      = 3,
   12.12      ParVerifyClaimValue        = 4,
   12.13      RebuildRSClaimValue        = 5,
   12.14 -    CompleteMarkCSetClaimValue = 6
   12.15 +    CompleteMarkCSetClaimValue = 6,
   12.16 +    ParEvacFailureClaimValue   = 7
   12.17    };
   12.18  
   12.19    inline HeapWord* par_allocate_no_bot_updates(size_t word_size) {
   12.20 @@ -582,37 +583,33 @@
   12.21    // that the collector is about to start or has finished (concurrently)
   12.22    // marking the heap.
   12.23  
   12.24 -  // Note the start of a marking phase. Record the
   12.25 -  // start of the unmarked area of the region here.
   12.26 -  void note_start_of_marking(bool during_initial_mark) {
   12.27 -    init_top_at_conc_mark_count();
   12.28 -    _next_marked_bytes = 0;
   12.29 -    if (during_initial_mark && is_young() && !is_survivor())
   12.30 -      _next_top_at_mark_start = bottom();
   12.31 -    else
   12.32 -      _next_top_at_mark_start = top();
   12.33 -  }
   12.34 +  // Notify the region that concurrent marking is starting. Initialize
   12.35 +  // all fields related to the next marking info.
   12.36 +  inline void note_start_of_marking();
   12.37  
   12.38 -  // Note the end of a marking phase. Install the start of
   12.39 -  // the unmarked area that was captured at start of marking.
   12.40 -  void note_end_of_marking() {
   12.41 -    _prev_top_at_mark_start = _next_top_at_mark_start;
   12.42 -    _prev_marked_bytes = _next_marked_bytes;
   12.43 -    _next_marked_bytes = 0;
   12.44 +  // Notify the region that concurrent marking has finished. Copy the
   12.45 +  // (now finalized) next marking info fields into the prev marking
   12.46 +  // info fields.
   12.47 +  inline void note_end_of_marking();
   12.48  
   12.49 -    guarantee(_prev_marked_bytes <=
   12.50 -              (size_t) (prev_top_at_mark_start() - bottom()) * HeapWordSize,
   12.51 -              "invariant");
   12.52 -  }
   12.53 +  // Notify the region that it will be used as to-space during a GC
   12.54 +  // and we are about to start copying objects into it.
   12.55 +  inline void note_start_of_copying(bool during_initial_mark);
   12.56  
   12.57 -  // After an evacuation, we need to update _next_top_at_mark_start
   12.58 -  // to be the current top.  Note this is only valid if we have only
   12.59 -  // ever evacuated into this region.  If we evacuate, allocate, and
   12.60 -  // then evacuate we are in deep doodoo.
   12.61 -  void note_end_of_copying() {
   12.62 -    assert(top() >= _next_top_at_mark_start, "Increase only");
   12.63 -    _next_top_at_mark_start = top();
   12.64 -  }
   12.65 +  // Notify the region that it ceases being to-space during a GC and
   12.66 +  // we will not copy objects into it any more.
   12.67 +  inline void note_end_of_copying(bool during_initial_mark);
   12.68 +
   12.69 +  // Notify the region that we are about to start processing
   12.70 +  // self-forwarded objects during evac failure handling.
   12.71 +  void note_self_forwarding_removal_start(bool during_initial_mark,
   12.72 +                                          bool during_conc_mark);
   12.73 +
   12.74 +  // Notify the region that we have finished processing self-forwarded
   12.75 +  // objects during evac failure handling.
   12.76 +  void note_self_forwarding_removal_end(bool during_initial_mark,
   12.77 +                                        bool during_conc_mark,
   12.78 +                                        size_t marked_bytes);
   12.79  
   12.80    // Returns "false" iff no object in the region was allocated when the
   12.81    // last mark phase ended.

    13.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.inline.hpp	Thu Jan 12 09:59:55 2012 -0500
    13.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.inline.hpp	Fri Jan 13 01:55:22 2012 -0800
    13.3 @@ -1,5 +1,5 @@
    13.4  /*
    13.5 - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
    13.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
    13.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    13.8   *
    13.9   * This code is free software; you can redistribute it and/or modify it
   13.10 @@ -55,4 +55,71 @@
   13.11    return _offsets.block_start_const(p);
   13.12  }
   13.13  
   13.14 +inline void HeapRegion::note_start_of_marking() {
   13.15 +  init_top_at_conc_mark_count();
   13.16 +  _next_marked_bytes = 0;
   13.17 +  _next_top_at_mark_start = top();
   13.18 +}
   13.19 +
   13.20 +inline void HeapRegion::note_end_of_marking() {
   13.21 +  _prev_top_at_mark_start = _next_top_at_mark_start;
   13.22 +  _prev_marked_bytes = _next_marked_bytes;
   13.23 +  _next_marked_bytes = 0;
   13.24 +
   13.25 +  assert(_prev_marked_bytes <=
   13.26 +         (size_t) pointer_delta(prev_top_at_mark_start(), bottom()) *
   13.27 +         HeapWordSize, "invariant");
   13.28 +}
   13.29 +
   13.30 +inline void HeapRegion::note_start_of_copying(bool during_initial_mark) {
   13.31 +  if (during_initial_mark) {
   13.32 +    if (is_survivor()) {
   13.33 +      assert(false, "should not allocate survivors during IM");
   13.34 +    } else {
   13.35 +      // During initial-mark we'll explicitly mark any objects on old
   13.36 +      // regions that are pointed to by roots. Given that explicit
   13.37 +      // marks only make sense under NTAMS it'd be nice if we could
   13.38 +      // check that condition if we wanted to. Given that we don't
   13.39 +      // know where the top of this region will end up, we simply set
   13.40 +      // NTAMS to the end of the region so all marks will be below
   13.41 +      // NTAMS. We'll set it to the actual top when we retire this region.
   13.42 +      _next_top_at_mark_start = end();
   13.43 +    }
   13.44 +  } else {
   13.45 +    if (is_survivor()) {
   13.46 +      // This is how we always allocate survivors.
   13.47 +      assert(_next_top_at_mark_start == bottom(), "invariant");
   13.48 +    } else {
   13.49 +      // We could have re-used this old region as to-space over a
   13.50 +      // couple of GCs since the start of the concurrent marking
   13.51 +      // cycle. This means that [bottom,NTAMS) will contain objects
   13.52 +      // copied up to and including initial-mark and [NTAMS, top)
   13.53 +      // will contain objects copied during the concurrent marking cycle.
   13.54 +      assert(top() >= _next_top_at_mark_start, "invariant");
   13.55 +    }
   13.56 +  }
   13.57 +}
   13.58 +
   13.59 +inline void HeapRegion::note_end_of_copying(bool during_initial_mark) {
   13.60 +  if (during_initial_mark) {
   13.61 +    if (is_survivor()) {
   13.62 +      assert(false, "should not allocate survivors during IM");
   13.63 +    } else {
   13.64 +      // See the comment for note_start_of_copying() for the details
   13.65 +      // on this.
   13.66 +      assert(_next_top_at_mark_start == end(), "pre-condition");
   13.67 +      _next_top_at_mark_start = top();
   13.68 +    }
   13.69 +  } else {
   13.70 +    if (is_survivor()) {
   13.71 +      // This is how we always allocate survivors.
   13.72 +      assert(_next_top_at_mark_start == bottom(), "invariant");
   13.73 +    } else {
   13.74 +      // See the comment for note_start_of_copying() for the details
   13.75 +      // on this.
   13.76 +      assert(top() >= _next_top_at_mark_start, "invariant");
   13.77 +    }
   13.78 +  }
   13.79 +}
   13.80 +
   13.81  #endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGION_INLINE_HPP

    14.1 --- a/src/share/vm/gc_implementation/g1/ptrQueue.hpp	Thu Jan 12 09:59:55 2012 -0500
    14.2 +++ b/src/share/vm/gc_implementation/g1/ptrQueue.hpp	Fri Jan 13 01:55:22 2012 -0800
    14.3 @@ -1,5 +1,5 @@
    14.4  /*
    14.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
    14.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
    14.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    14.8   *
    14.9   * This code is free software; you can redistribute it and/or modify it
   14.10 @@ -70,7 +70,7 @@
   14.11    // given PtrQueueSet.
   14.12    PtrQueue(PtrQueueSet* qset, bool perm = false, bool active = false);
   14.13    // Release any contained resources.
   14.14 -  void flush();
   14.15 +  virtual void flush();
   14.16    // Calls flush() when destroyed.
   14.17    ~PtrQueue() { flush(); }
   14.18  

    15.1 --- a/src/share/vm/gc_implementation/g1/satbQueue.cpp	Thu Jan 12 09:59:55 2012 -0500
    15.2 +++ b/src/share/vm/gc_implementation/g1/satbQueue.cpp	Fri Jan 13 01:55:22 2012 -0800
    15.3 @@ -1,5 +1,5 @@
    15.4  /*
    15.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
    15.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
    15.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    15.8   *
    15.9   * This code is free software; you can redistribute it and/or modify it
   15.10 @@ -31,6 +31,14 @@
   15.11  #include "runtime/thread.hpp"
   15.12  #include "runtime/vmThread.hpp"
   15.13  
   15.14 +void ObjPtrQueue::flush() {
   15.15 +  // The buffer might contain refs into the CSet. We have to filter it
   15.16 +  // first before we flush it, otherwise we might end up with an
   15.17 +  // enqueued buffer with refs into the CSet which breaks our invariants.
   15.18 +  filter();
   15.19 +  PtrQueue::flush();
   15.20 +}
   15.21 +
   15.22  // This method removes entries from an SATB buffer that will not be
   15.23  // useful to the concurrent marking threads. An entry is removed if it
   15.24  // satisfies one of the following conditions:
   15.25 @@ -44,38 +52,27 @@
   15.26  //     process it again).
   15.27  //
   15.28  // The rest of the entries will be retained and are compacted towards
   15.29 -// the top of the buffer. If with this filtering we clear a large
   15.30 -// enough chunk of the buffer we can re-use it (instead of enqueueing
   15.31 -// it) and we can just allow the mutator to carry on executing.
   15.32 +// the top of the buffer. Note that, because we do not allow old
   15.33 +// regions in the CSet during marking, all objects on the CSet regions
   15.34 +// are young (eden or survivors) and therefore implicitly live. So any
   15.35 +// references into the CSet will be removed during filtering.
   15.36  
   15.37 -bool ObjPtrQueue::should_enqueue_buffer() {
   15.38 -  assert(_lock == NULL || _lock->owned_by_self(),
   15.39 -         "we should have taken the lock before calling this");
   15.40 -
   15.41 -  // A value of 0 means "don't filter SATB buffers".
   15.42 -  if (G1SATBBufferEnqueueingThresholdPercent == 0) {
   15.43 -    return true;
   15.44 -  }
   15.45 -
   15.46 +void ObjPtrQueue::filter() {
   15.47    G1CollectedHeap* g1h = G1CollectedHeap::heap();
   15.48 -
   15.49 -  // This method should only be called if there is a non-NULL buffer
   15.50 -  // that is full.
   15.51 -  assert(_index == 0, "pre-condition");
   15.52 -  assert(_buf != NULL, "pre-condition");
   15.53 -
   15.54    void** buf = _buf;
   15.55    size_t sz = _sz;
   15.56  
   15.57 +  if (buf == NULL) {
   15.58 +    // nothing to do
   15.59 +    return;
   15.60 +  }
   15.61 +
   15.62    // Used for sanity checking at the end of the loop.
   15.63    debug_only(size_t entries = 0; size_t retained = 0;)
   15.64  
   15.65    size_t i = sz;
   15.66    size_t new_index = sz;
   15.67  
   15.68 -  // Given that we are expecting _index == 0, we could have changed
   15.69 -  // the loop condition to (i > 0). But we are using _index for
   15.70 -  // generality.
   15.71    while (i > _index) {
   15.72      assert(i > 0, "we should have at least one more entry to process");
   15.73      i -= oopSize;
   15.74 @@ -103,22 +100,58 @@
   15.75        debug_only(retained += 1;)
   15.76      }
   15.77    }
   15.78 +
   15.79 +#ifdef ASSERT
   15.80    size_t entries_calc = (sz - _index) / oopSize;
   15.81    assert(entries == entries_calc, "the number of entries we counted "
   15.82           "should match the number of entries we calculated");
   15.83    size_t retained_calc = (sz - new_index) / oopSize;
   15.84    assert(retained == retained_calc, "the number of retained entries we counted "
   15.85           "should match the number of retained entries we calculated");
   15.86 -  size_t perc = retained_calc * 100 / entries_calc;
   15.87 +#endif // ASSERT
   15.88 +
   15.89 +  _index = new_index;
   15.90 +}
   15.91 +
   15.92 +// This method will first apply the above filtering to the buffer. If
   15.93 +// post-filtering a large enough chunk of the buffer has been cleared
   15.94 +// we can re-use the buffer (instead of enqueueing it) and we can just
   15.95 +// allow the mutator to carry on executing using the same buffer
   15.96 +// instead of replacing it.
   15.97 +
   15.98 +bool ObjPtrQueue::should_enqueue_buffer() {
   15.99 +  assert(_lock == NULL || _lock->owned_by_self(),
  15.100 +         "we should have taken the lock before calling this");
  15.101 +
  15.102 +  // Even if G1SATBBufferEnqueueingThresholdPercent == 0 we have to
  15.103 +  // filter the buffer given that this will remove any references into
  15.104 +  // the CSet as we currently assume that no such refs will appear in
  15.105 +  // enqueued buffers.
  15.106 +
  15.107 +  // This method should only be called if there is a non-NULL buffer
  15.108 +  // that is full.
  15.109 +  assert(_index == 0, "pre-condition");
  15.110 +  assert(_buf != NULL, "pre-condition");
  15.111 +
  15.112 +  filter();
  15.113 +
  15.114 +  size_t sz = _sz;
  15.115 +  size_t all_entries = sz / oopSize;
  15.116 +  size_t retained_entries = (sz - _index) / oopSize;
  15.117 +  size_t perc = retained_entries * 100 / all_entries;
  15.118    bool should_enqueue = perc > (size_t) G1SATBBufferEnqueueingThresholdPercent;
  15.119 -  _index = new_index;
  15.120 -
  15.121    return should_enqueue;
  15.122  }
  15.123  
  15.124  void ObjPtrQueue::apply_closure(ObjectClosure* cl) {
  15.125    if (_buf != NULL) {
  15.126      apply_closure_to_buffer(cl, _buf, _index, _sz);
  15.127 +  }
  15.128 +}
  15.129 +
  15.130 +void ObjPtrQueue::apply_closure_and_empty(ObjectClosure* cl) {
  15.131 +  if (_buf != NULL) {
  15.132 +    apply_closure_to_buffer(cl, _buf, _index, _sz);
  15.133      _index = _sz;
  15.134    }
  15.135  }
  15.136 @@ -135,6 +168,21 @@
  15.137    }
  15.138  }
  15.139  
  15.140 +#ifndef PRODUCT
  15.141 +// Helpful for debugging
  15.142 +
  15.143 +void ObjPtrQueue::print(const char* name) {
  15.144 +  print(name, _buf, _index, _sz);
  15.145 +}
  15.146 +
  15.147 +void ObjPtrQueue::print(const char* name,
  15.148 +                        void** buf, size_t index, size_t sz) {
  15.149 +  gclog_or_tty->print_cr("  SATB BUFFER [%s] buf: "PTR_FORMAT" "
  15.150 +                         "index: "SIZE_FORMAT" sz: "SIZE_FORMAT,
  15.151 +                         name, buf, index, sz);
  15.152 +}
  15.153 +#endif // PRODUCT
  15.154 +
  15.155  #ifdef ASSERT
  15.156  void ObjPtrQueue::verify_oops_in_buffer() {
  15.157    if (_buf == NULL) return;
  15.158 @@ -150,12 +198,9 @@
  15.159  #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
  15.160  #endif // _MSC_VER
  15.161  
  15.162 -
  15.163  SATBMarkQueueSet::SATBMarkQueueSet() :
  15.164 -  PtrQueueSet(),
  15.165 -  _closure(NULL), _par_closures(NULL),
  15.166 -  _shared_satb_queue(this, true /*perm*/)
  15.167 -{}
  15.168 +  PtrQueueSet(), _closure(NULL), _par_closures(NULL),
  15.169 +  _shared_satb_queue(this, true /*perm*/) { }
  15.170  
  15.171  void SATBMarkQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock,
  15.172                                    int process_completed_threshold,
  15.173 @@ -167,7 +212,6 @@
  15.174    }
  15.175  }
  15.176  
  15.177 -
  15.178  void SATBMarkQueueSet::handle_zero_index_for_thread(JavaThread* t) {
  15.179    DEBUG_ONLY(t->satb_mark_queue().verify_oops_in_buffer();)
  15.180    t->satb_mark_queue().handle_zero_index();
  15.181 @@ -228,6 +272,13 @@
  15.182    }
  15.183  }
  15.184  
  15.185 +void SATBMarkQueueSet::filter_thread_buffers() {
  15.186 +  for(JavaThread* t = Threads::first(); t; t = t->next()) {
  15.187 +    t->satb_mark_queue().filter();
  15.188 +  }
  15.189 +  shared_satb_queue()->filter();
  15.190 +}
  15.191 +
  15.192  void SATBMarkQueueSet::set_closure(ObjectClosure* closure) {
  15.193    _closure = closure;
  15.194  }
  15.195 @@ -239,9 +290,9 @@
  15.196  
  15.197  void SATBMarkQueueSet::iterate_closure_all_threads() {
  15.198    for(JavaThread* t = Threads::first(); t; t = t->next()) {
  15.199 -    t->satb_mark_queue().apply_closure(_closure);
  15.200 +    t->satb_mark_queue().apply_closure_and_empty(_closure);
  15.201    }
  15.202 -  shared_satb_queue()->apply_closure(_closure);
  15.203 +  shared_satb_queue()->apply_closure_and_empty(_closure);
  15.204  }
  15.205  
  15.206  void SATBMarkQueueSet::par_iterate_closure_all_threads(int worker) {
  15.207 @@ -250,7 +301,7 @@
  15.208  
  15.209    for(JavaThread* t = Threads::first(); t; t = t->next()) {
  15.210      if (t->claim_oops_do(true, parity)) {
  15.211 -      t->satb_mark_queue().apply_closure(_par_closures[worker]);
  15.212 +      t->satb_mark_queue().apply_closure_and_empty(_par_closures[worker]);
  15.213      }
  15.214    }
  15.215  
  15.216 @@ -264,7 +315,7 @@
  15.217  
  15.218    VMThread* vmt = VMThread::vm_thread();
  15.219    if (vmt->claim_oops_do(true, parity)) {
  15.220 -    shared_satb_queue()->apply_closure(_par_closures[worker]);
  15.221 +    shared_satb_queue()->apply_closure_and_empty(_par_closures[worker]);
  15.222    }
  15.223  }
  15.224  
  15.225 @@ -292,6 +343,61 @@
  15.226    }
  15.227  }
  15.228  
  15.229 +void SATBMarkQueueSet::iterate_completed_buffers_read_only(ObjectClosure* cl) {
  15.230 +  assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
  15.231 +  assert(cl != NULL, "pre-condition");
  15.232 +
  15.233 +  BufferNode* nd = _completed_buffers_head;
  15.234 +  while (nd != NULL) {
  15.235 +    void** buf = BufferNode::make_buffer_from_node(nd);
  15.236 +    ObjPtrQueue::apply_closure_to_buffer(cl, buf, 0, _sz);
  15.237 +    nd = nd->next();
  15.238 +  }
  15.239 +}
  15.240 +
  15.241 +void SATBMarkQueueSet::iterate_thread_buffers_read_only(ObjectClosure* cl) {
  15.242 +  assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
  15.243 +  assert(cl != NULL, "pre-condition");
  15.244 +
  15.245 +  for (JavaThread* t = Threads::first(); t; t = t->next()) {
  15.246 +    t->satb_mark_queue().apply_closure(cl);
  15.247 +  }
  15.248 +  shared_satb_queue()->apply_closure(cl);
  15.249 +}
  15.250 +
  15.251 +#ifndef PRODUCT
  15.252 +// Helpful for debugging
  15.253 +
  15.254 +#define SATB_PRINTER_BUFFER_SIZE 256
  15.255 +
  15.256 +void SATBMarkQueueSet::print_all(const char* msg) {
  15.257 +  char buffer[SATB_PRINTER_BUFFER_SIZE];
  15.258 +  assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
  15.259 +
  15.260 +  gclog_or_tty->cr();
  15.261 +  gclog_or_tty->print_cr("SATB BUFFERS [%s]", msg);
  15.262 +
  15.263 +  BufferNode* nd = _completed_buffers_head;
  15.264 +  int i = 0;
  15.265 +  while (nd != NULL) {
  15.266 +    void** buf = BufferNode::make_buffer_from_node(nd);
  15.267 +    jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Enqueued: %d", i);
  15.268 +    ObjPtrQueue::print(buffer, buf, 0, _sz);
  15.269 +    nd = nd->next();
  15.270 +    i += 1;
  15.271 +  }
  15.272 +
  15.273 +  for (JavaThread* t = Threads::first(); t; t = t->next()) {
  15.274 +    jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Thread: %s", t->name());
  15.275 +    t->satb_mark_queue().print(buffer);
  15.276 +  }
  15.277 +
  15.278 +  shared_satb_queue()->print("Shared");
  15.279 +
  15.280 +  gclog_or_tty->cr();
  15.281 +}
  15.282 +#endif // PRODUCT
  15.283 +
  15.284  void SATBMarkQueueSet::abandon_partial_marking() {
  15.285    BufferNode* buffers_to_delete = NULL;
  15.286    {
  15.287 @@ -316,5 +422,5 @@
  15.288    for (JavaThread* t = Threads::first(); t; t = t->next()) {
  15.289      t->satb_mark_queue().reset();
  15.290    }
  15.291 -  shared_satb_queue()->reset();
  15.292 + shared_satb_queue()->reset();
  15.293  }

    16.1 --- a/src/share/vm/gc_implementation/g1/satbQueue.hpp	Thu Jan 12 09:59:55 2012 -0500
    16.2 +++ b/src/share/vm/gc_implementation/g1/satbQueue.hpp	Fri Jan 13 01:55:22 2012 -0800
    16.3 @@ -1,5 +1,5 @@
    16.4  /*
    16.5 - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
    16.6 + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
    16.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    16.8   *
    16.9   * This code is free software; you can redistribute it and/or modify it
   16.10 @@ -29,9 +29,26 @@
   16.11  
   16.12  class ObjectClosure;
   16.13  class JavaThread;
   16.14 +class SATBMarkQueueSet;
   16.15  
   16.16  // A ptrQueue whose elements are "oops", pointers to object heads.
   16.17  class ObjPtrQueue: public PtrQueue {
   16.18 +  friend class SATBMarkQueueSet;
   16.19 +
   16.20 +private:
   16.21 +  // Filter out unwanted entries from the buffer.
   16.22 +  void filter();
   16.23 +
   16.24 +  // Apply the closure to all elements.
   16.25 +  void apply_closure(ObjectClosure* cl);
   16.26 +
   16.27 +  // Apply the closure to all elements and empty the buffer;
   16.28 +  void apply_closure_and_empty(ObjectClosure* cl);
   16.29 +
   16.30 +  // Apply the closure to all elements of "buf", down to "index" (inclusive.)
   16.31 +  static void apply_closure_to_buffer(ObjectClosure* cl,
   16.32 +                                      void** buf, size_t index, size_t sz);
   16.33 +
   16.34  public:
   16.35    ObjPtrQueue(PtrQueueSet* qset, bool perm = false) :
   16.36      // SATB queues are only active during marking cycles. We create
   16.37 @@ -41,23 +58,23 @@
   16.38      // field to true. This is done in JavaThread::initialize_queues().
   16.39      PtrQueue(qset, perm, false /* active */) { }
   16.40  
   16.41 +  // Overrides PtrQueue::flush() so that it can filter the buffer
   16.42 +  // before it is flushed.
   16.43 +  virtual void flush();
   16.44 +
   16.45    // Overrides PtrQueue::should_enqueue_buffer(). See the method's
   16.46    // definition for more information.
   16.47    virtual bool should_enqueue_buffer();
   16.48  
   16.49 -  // Apply the closure to all elements, and reset the index to make the
   16.50 -  // buffer empty.
   16.51 -  void apply_closure(ObjectClosure* cl);
   16.52 -
   16.53 -  // Apply the closure to all elements of "buf", down to "index" (inclusive.)
   16.54 -  static void apply_closure_to_buffer(ObjectClosure* cl,
   16.55 -                                      void** buf, size_t index, size_t sz);
   16.56 +#ifndef PRODUCT
   16.57 +  // Helpful for debugging
   16.58 +  void print(const char* name);
   16.59 +  static void print(const char* name, void** buf, size_t index, size_t sz);
   16.60 +#endif // PRODUCT
   16.61  
   16.62    void verify_oops_in_buffer() NOT_DEBUG_RETURN;
   16.63  };
   16.64  
   16.65 -
   16.66 -
   16.67  class SATBMarkQueueSet: public PtrQueueSet {
   16.68    ObjectClosure* _closure;
   16.69    ObjectClosure** _par_closures;  // One per ParGCThread.
   16.70 @@ -88,6 +105,9 @@
   16.71    // set itself, has an active value same as expected_active.
   16.72    void set_active_all_threads(bool b, bool expected_active);
   16.73  
   16.74 +  // Filter all the currently-active SATB buffers.
   16.75 +  void filter_thread_buffers();
   16.76 +
   16.77    // Register "blk" as "the closure" for all queues.  Only one such closure
   16.78    // is allowed.  The "apply_closure_to_completed_buffer" method will apply
   16.79    // this closure to a completed buffer, and "iterate_closure_all_threads"
   16.80 @@ -98,10 +118,9 @@
   16.81    // closures, one for each parallel GC thread.
   16.82    void set_par_closure(int i, ObjectClosure* closure);
   16.83  
   16.84 -  // If there is a registered closure for buffers, apply it to all entries
   16.85 -  // in all currently-active buffers.  This should only be applied at a
   16.86 -  // safepoint.  (Currently must not be called in parallel; this should
   16.87 -  // change in the future.)
   16.88 +  // Apply the registered closure to all entries on each
   16.89 +  // currently-active buffer and then empty the buffer. It should only
   16.90 +  // be called serially and at a safepoint.
   16.91    void iterate_closure_all_threads();
   16.92    // Parallel version of the above.
   16.93    void par_iterate_closure_all_threads(int worker);
   16.94 @@ -117,11 +136,21 @@
   16.95      return apply_closure_to_completed_buffer_work(true, worker);
   16.96    }
   16.97  
   16.98 +  // Apply the given closure on enqueued and currently-active buffers
   16.99 +  // respectively. Both methods are read-only, i.e., they do not
  16.100 +  // modify any of the buffers.
  16.101 +  void iterate_completed_buffers_read_only(ObjectClosure* cl);
  16.102 +  void iterate_thread_buffers_read_only(ObjectClosure* cl);
  16.103 +
  16.104 +#ifndef PRODUCT
  16.105 +  // Helpful for debugging
  16.106 +  void print_all(const char* msg);
  16.107 +#endif // PRODUCT
  16.108 +
  16.109    ObjPtrQueue* shared_satb_queue() { return &_shared_satb_queue; }
  16.110  
  16.111    // If a marking is being abandoned, reset any unprocessed log buffers.
  16.112    void abandon_partial_marking();
  16.113 -
  16.114  };
  16.115  
  16.116  #endif // SHARE_VM_GC_IMPLEMENTATION_G1_SATBQUEUE_HPP