Mercurial > jdk8-mips64-public > hotspot / changeset

     1.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Wed Sep 08 16:10:51 2010 -0700
     1.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Tue Sep 28 09:51:37 2010 -0700
     1.3 @@ -1,5 +1,5 @@
     1.4  /*
     1.5 - * Copyright (c) 2001, 2009, Oracle and/or its affiliates. All rights reserved.
     1.6 + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
     1.7   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     1.8   *
     1.9   * This code is free software; you can redistribute it and/or modify it
    1.10 @@ -278,15 +278,16 @@
    1.11    if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base);
    1.12  }
    1.13  
    1.14 -void CMRegionStack::push(MemRegion mr) {
    1.15 +void CMRegionStack::push_lock_free(MemRegion mr) {
    1.16    assert(mr.word_size() > 0, "Precondition");
    1.17    while (true) {
    1.18 -    if (isFull()) {
    1.19 +    jint index = _index;
    1.20 +
    1.21 +    if (index >= _capacity) {
    1.22        _overflow = true;
    1.23        return;
    1.24      }
    1.25      // Otherwise...
    1.26 -    jint index = _index;
    1.27      jint next_index = index+1;
    1.28      jint res = Atomic::cmpxchg(next_index, &_index, index);
    1.29      if (res == index) {
    1.30 @@ -297,19 +298,17 @@
    1.31    }
    1.32  }
    1.33  
    1.34 -// Currently we do not call this at all. Normally we would call it
    1.35 -// during the concurrent marking / remark phases but we now call
    1.36 -// the lock-based version instead. But we might want to resurrect this
    1.37 -// code in the future. So, we'll leave it here commented out.
    1.38 -#if 0
    1.39 -MemRegion CMRegionStack::pop() {
    1.40 +// Lock-free pop of the region stack. Called during the concurrent
    1.41 +// marking / remark phases. Should only be called in tandem with
    1.42 +// other lock-free pops.
    1.43 +MemRegion CMRegionStack::pop_lock_free() {
    1.44    while (true) {
    1.45 -    // Otherwise...
    1.46      jint index = _index;
    1.47  
    1.48      if (index == 0) {
    1.49        return MemRegion();
    1.50      }
    1.51 +    // Otherwise...
    1.52      jint next_index = index-1;
    1.53      jint res = Atomic::cmpxchg(next_index, &_index, index);
    1.54      if (res == index) {
    1.55 @@ -326,7 +325,11 @@
    1.56      // Otherwise, we need to try again.
    1.57    }
    1.58  }
    1.59 -#endif // 0
    1.60 +
    1.61 +#if 0
    1.62 +// The routines that manipulate the region stack with a lock are
    1.63 +// not currently used. They should be retained, however, as a
    1.64 +// diagnostic aid.
    1.65  
    1.66  void CMRegionStack::push_with_lock(MemRegion mr) {
    1.67    assert(mr.word_size() > 0, "Precondition");
    1.68 @@ -361,6 +364,7 @@
    1.69      }
    1.70    }
    1.71  }
    1.72 +#endif
    1.73  
    1.74  bool CMRegionStack::invalidate_entries_into_cset() {
    1.75    bool result = false;
    1.76 @@ -648,8 +652,9 @@
    1.77    // We do reset all of them, since different phases will use
    1.78    // different number of active threads. So, it's easiest to have all
    1.79    // of them ready.
    1.80 -  for (int i = 0; i < (int) _max_task_num; ++i)
    1.81 +  for (int i = 0; i < (int) _max_task_num; ++i) {
    1.82      _tasks[i]->reset(_nextMarkBitMap);
    1.83 +  }
    1.84  
    1.85    // we need this to make sure that the flag is on during the evac
    1.86    // pause with initial mark piggy-backed
    1.87 @@ -988,7 +993,7 @@
    1.88                               "below the finger, pushing it",
    1.89                               mr.start(), mr.end());
    1.90  
    1.91 -    if (!region_stack_push(mr)) {
    1.92 +    if (!region_stack_push_lock_free(mr)) {
    1.93        if (verbose_low())
    1.94          gclog_or_tty->print_cr("[global] region stack has overflown.");
    1.95      }
    1.96 @@ -2333,6 +2338,39 @@
    1.97    return NULL;
    1.98  }
    1.99  
   1.100 +bool ConcurrentMark::invalidate_aborted_regions_in_cset() {
   1.101 +  bool result = false;
   1.102 +  for (int i = 0; i < (int)_max_task_num; ++i) {
   1.103 +    CMTask* the_task = _tasks[i];
   1.104 +    MemRegion mr = the_task->aborted_region();
   1.105 +    if (mr.start() != NULL) {
   1.106 +      assert(mr.end() != NULL, "invariant");
   1.107 +      assert(mr.word_size() > 0, "invariant");
   1.108 +      HeapRegion* hr = _g1h->heap_region_containing(mr.start());
   1.109 +      assert(hr != NULL, "invariant");
   1.110 +      if (hr->in_collection_set()) {
   1.111 +        // The region points into the collection set
   1.112 +        the_task->set_aborted_region(MemRegion());
   1.113 +        result = true;
   1.114 +      }
   1.115 +    }
   1.116 +  }
   1.117 +  return result;
   1.118 +}
   1.119 +
   1.120 +bool ConcurrentMark::has_aborted_regions() {
   1.121 +  for (int i = 0; i < (int)_max_task_num; ++i) {
   1.122 +    CMTask* the_task = _tasks[i];
   1.123 +    MemRegion mr = the_task->aborted_region();
   1.124 +    if (mr.start() != NULL) {
   1.125 +      assert(mr.end() != NULL, "invariant");
   1.126 +      assert(mr.word_size() > 0, "invariant");
   1.127 +      return true;
   1.128 +    }
   1.129 +  }
   1.130 +  return false;
   1.131 +}
   1.132 +
   1.133  void ConcurrentMark::oops_do(OopClosure* cl) {
   1.134    if (_markStack.size() > 0 && verbose_low())
   1.135      gclog_or_tty->print_cr("[global] scanning the global marking stack, "
   1.136 @@ -2351,13 +2389,22 @@
   1.137      queue->oops_do(cl);
   1.138    }
   1.139  
   1.140 -  // finally, invalidate any entries that in the region stack that
   1.141 +  // Invalidate any entries, that are in the region stack, that
   1.142    // point into the collection set
   1.143    if (_regionStack.invalidate_entries_into_cset()) {
   1.144      // otherwise, any gray objects copied during the evacuation pause
   1.145      // might not be visited.
   1.146      assert(_should_gray_objects, "invariant");
   1.147    }
   1.148 +
   1.149 +  // Invalidate any aborted regions, recorded in the individual CM
   1.150 +  // tasks, that point into the collection set.
   1.151 +  if (invalidate_aborted_regions_in_cset()) {
   1.152 +    // otherwise, any gray objects copied during the evacuation pause
   1.153 +    // might not be visited.
   1.154 +    assert(_should_gray_objects, "invariant");
   1.155 +  }
   1.156 +
   1.157  }
   1.158  
   1.159  void ConcurrentMark::clear_marking_state() {
   1.160 @@ -2638,7 +2685,7 @@
   1.161    // irrespective whether all collection set regions are below the
   1.162    // finger, if the region stack is not empty. This is expected to be
   1.163    // a rare case, so I don't think it's necessary to be smarted about it.
   1.164 -  if (!region_stack_empty())
   1.165 +  if (!region_stack_empty() || has_aborted_regions())
   1.166      _should_gray_objects = true;
   1.167  }
   1.168  
   1.169 @@ -2657,8 +2704,10 @@
   1.170    _nextMarkBitMap->clearAll();
   1.171    // Empty mark stack
   1.172    clear_marking_state();
   1.173 -  for (int i = 0; i < (int)_max_task_num; ++i)
   1.174 +  for (int i = 0; i < (int)_max_task_num; ++i) {
   1.175      _tasks[i]->clear_region_fields();
   1.176 +    _tasks[i]->clear_aborted_region();
   1.177 +  }
   1.178    _has_aborted = true;
   1.179  
   1.180    SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
   1.181 @@ -2936,6 +2985,7 @@
   1.182  
   1.183    _nextMarkBitMap                = nextMarkBitMap;
   1.184    clear_region_fields();
   1.185 +  clear_aborted_region();
   1.186  
   1.187    _calls                         = 0;
   1.188    _elapsed_time_ms               = 0.0;
   1.189 @@ -3428,20 +3478,32 @@
   1.190    assert(_region_finger == NULL,
   1.191           "it should be NULL when we're not scanning a region");
   1.192  
   1.193 -  if (!_cm->region_stack_empty()) {
   1.194 +  if (!_cm->region_stack_empty() || !_aborted_region.is_empty()) {
   1.195      if (_cm->verbose_low())
   1.196        gclog_or_tty->print_cr("[%d] draining region stack, size = %d",
   1.197                               _task_id, _cm->region_stack_size());
   1.198  
   1.199 -    MemRegion mr = _cm->region_stack_pop_with_lock();
   1.200 -    // it returns MemRegion() if the pop fails
   1.201 -    statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
   1.202 +    MemRegion mr;
   1.203 +
   1.204 +    if (!_aborted_region.is_empty()) {
   1.205 +      mr = _aborted_region;
   1.206 +      _aborted_region = MemRegion();
   1.207 +
   1.208 +      if (_cm->verbose_low())
   1.209 +        gclog_or_tty->print_cr("[%d] scanning aborted region [ " PTR_FORMAT ", " PTR_FORMAT " )",
   1.210 +                             _task_id, mr.start(), mr.end());
   1.211 +    } else {
   1.212 +      mr = _cm->region_stack_pop_lock_free();
   1.213 +      // it returns MemRegion() if the pop fails
   1.214 +      statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
   1.215 +    }
   1.216  
   1.217      while (mr.start() != NULL) {
   1.218        if (_cm->verbose_medium())
   1.219          gclog_or_tty->print_cr("[%d] we are scanning region "
   1.220                                 "["PTR_FORMAT", "PTR_FORMAT")",
   1.221                                 _task_id, mr.start(), mr.end());
   1.222 +
   1.223        assert(mr.end() <= _cm->finger(),
   1.224               "otherwise the region shouldn't be on the stack");
   1.225        assert(!mr.is_empty(), "Only non-empty regions live on the region stack");
   1.226 @@ -3454,7 +3516,7 @@
   1.227          if (has_aborted())
   1.228            mr = MemRegion();
   1.229          else {
   1.230 -          mr = _cm->region_stack_pop_with_lock();
   1.231 +          mr = _cm->region_stack_pop_lock_free();
   1.232            // it returns MemRegion() if the pop fails
   1.233            statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
   1.234          }
   1.235 @@ -3468,6 +3530,10 @@
   1.236          // have definitely set _region_finger to something non-null.
   1.237          assert(_region_finger != NULL, "invariant");
   1.238  
   1.239 +        // Make sure that any previously aborted region has been
   1.240 +        // cleared.
   1.241 +        assert(_aborted_region.is_empty(), "aborted region not cleared");
   1.242 +
   1.243          // The iteration was actually aborted. So now _region_finger
   1.244          // points to the address of the object we last scanned. If we
   1.245          // leave it there, when we restart this task, we will rescan
   1.246 @@ -3480,14 +3546,14 @@
   1.247  
   1.248          if (!newRegion.is_empty()) {
   1.249            if (_cm->verbose_low()) {
   1.250 -            gclog_or_tty->print_cr("[%d] pushing unscanned region"
   1.251 -                                   "[" PTR_FORMAT "," PTR_FORMAT ") on region stack",
   1.252 +            gclog_or_tty->print_cr("[%d] recording unscanned region"
   1.253 +                                   "[" PTR_FORMAT "," PTR_FORMAT ") in CMTask",
   1.254                                     _task_id,
   1.255                                     newRegion.start(), newRegion.end());
   1.256            }
   1.257 -          // Now push the part of the region we didn't scan on the
   1.258 -          // region stack to make sure a task scans it later.
   1.259 -          _cm->region_stack_push_with_lock(newRegion);
   1.260 +          // Now record the part of the region we didn't scan to
   1.261 +          // make sure this task scans it later.
   1.262 +          _aborted_region = newRegion;
   1.263          }
   1.264          // break from while
   1.265          mr = MemRegion();
   1.266 @@ -3657,6 +3723,8 @@
   1.267  
   1.268    assert(concurrent() || _cm->region_stack_empty(),
   1.269           "the region stack should have been cleared before remark");
   1.270 +  assert(concurrent() || !_cm->has_aborted_regions(),
   1.271 +         "aborted regions should have been cleared before remark");
   1.272    assert(_region_finger == NULL,
   1.273           "this should be non-null only when a region is being scanned");
   1.274  
   1.275 @@ -3946,6 +4014,7 @@
   1.276        // that, if a condition is false, we can immediately find out
   1.277        // which one.
   1.278        guarantee(_cm->out_of_regions(), "only way to reach here");
   1.279 +      guarantee(_aborted_region.is_empty(), "only way to reach here");
   1.280        guarantee(_cm->region_stack_empty(), "only way to reach here");
   1.281        guarantee(_cm->mark_stack_empty(), "only way to reach here");
   1.282        guarantee(_task_queue->size() == 0, "only way to reach here");
   1.283 @@ -4045,7 +4114,8 @@
   1.284      _nextMarkBitMap(NULL), _hash_seed(17),
   1.285      _task_queue(task_queue),
   1.286      _task_queues(task_queues),
   1.287 -    _oop_closure(NULL) {
   1.288 +    _oop_closure(NULL),
   1.289 +    _aborted_region(MemRegion()) {
   1.290    guarantee(task_queue != NULL, "invariant");
   1.291    guarantee(task_queues != NULL, "invariant");
   1.292  

     2.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Wed Sep 08 16:10:51 2010 -0700
     2.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Tue Sep 28 09:51:37 2010 -0700
     2.3 @@ -250,21 +250,23 @@
     2.4  
     2.5    // This is lock-free; assumes that it will only be called in parallel
     2.6    // with other "push" operations (no pops).
     2.7 -  void push(MemRegion mr);
     2.8 -
     2.9 -#if 0
    2.10 -  // This is currently not used. See the comment in the .cpp file.
    2.11 +  void push_lock_free(MemRegion mr);
    2.12  
    2.13    // Lock-free; assumes that it will only be called in parallel
    2.14    // with other "pop" operations (no pushes).
    2.15 -  MemRegion pop();
    2.16 -#endif // 0
    2.17 +  MemRegion pop_lock_free();
    2.18 +
    2.19 +#if 0
    2.20 +  // The routines that manipulate the region stack with a lock are
    2.21 +  // not currently used. They should be retained, however, as a
    2.22 +  // diagnostic aid.
    2.23  
    2.24    // These two are the implementations that use a lock. They can be
    2.25    // called concurrently with each other but they should not be called
    2.26    // concurrently with the lock-free versions (push() / pop()).
    2.27    void push_with_lock(MemRegion mr);
    2.28    MemRegion pop_with_lock();
    2.29 +#endif
    2.30  
    2.31    bool isEmpty()    { return _index == 0; }
    2.32    bool isFull()     { return _index == _capacity; }
    2.33 @@ -398,6 +400,7 @@
    2.34    volatile bool           _concurrent;
    2.35    // set at the end of a Full GC so that marking aborts
    2.36    volatile bool           _has_aborted;
    2.37 +
    2.38    // used when remark aborts due to an overflow to indicate that
    2.39    // another concurrent marking phase should start
    2.40    volatile bool           _restart_for_overflow;
    2.41 @@ -548,23 +551,30 @@
    2.42    bool mark_stack_overflow()            { return _markStack.overflow(); }
    2.43    bool mark_stack_empty()               { return _markStack.isEmpty(); }
    2.44  
    2.45 -  // Manipulation of the region stack
    2.46 -  bool region_stack_push(MemRegion mr) {
    2.47 +  // (Lock-free) Manipulation of the region stack
    2.48 +  bool region_stack_push_lock_free(MemRegion mr) {
    2.49      // Currently we only call the lock-free version during evacuation
    2.50      // pauses.
    2.51      assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
    2.52  
    2.53 -    _regionStack.push(mr);
    2.54 +    _regionStack.push_lock_free(mr);
    2.55      if (_regionStack.overflow()) {
    2.56        set_has_overflown();
    2.57        return false;
    2.58      }
    2.59      return true;
    2.60    }
    2.61 +
    2.62 +  // Lock-free version of region-stack pop. Should only be
    2.63 +  // called in tandem with other lock-free pops.
    2.64 +  MemRegion region_stack_pop_lock_free() {
    2.65 +    return _regionStack.pop_lock_free();
    2.66 +  }
    2.67 +
    2.68  #if 0
    2.69 -  // Currently this is not used. See the comment in the .cpp file.
    2.70 -  MemRegion region_stack_pop() { return _regionStack.pop(); }
    2.71 -#endif // 0
    2.72 +  // The routines that manipulate the region stack with a lock are
    2.73 +  // not currently used. They should be retained, however, as a
    2.74 +  // diagnostic aid.
    2.75  
    2.76    bool region_stack_push_with_lock(MemRegion mr) {
    2.77      // Currently we only call the lock-based version during either
    2.78 @@ -579,6 +589,7 @@
    2.79      }
    2.80      return true;
    2.81    }
    2.82 +
    2.83    MemRegion region_stack_pop_with_lock() {
    2.84      // Currently we only call the lock-based version during either
    2.85      // concurrent marking or remark.
    2.86 @@ -587,11 +598,21 @@
    2.87  
    2.88      return _regionStack.pop_with_lock();
    2.89    }
    2.90 +#endif
    2.91  
    2.92    int region_stack_size()               { return _regionStack.size(); }
    2.93    bool region_stack_overflow()          { return _regionStack.overflow(); }
    2.94    bool region_stack_empty()             { return _regionStack.isEmpty(); }
    2.95  
    2.96 +  // Iterate over any regions that were aborted while draining the
    2.97 +  // region stack (any such regions are saved in the corresponding
    2.98 +  // CMTask) and invalidate (i.e. assign to the empty MemRegion())
    2.99 +  // any regions that point into the collection set.
   2.100 +  bool invalidate_aborted_regions_in_cset();
   2.101 +
   2.102 +  // Returns true if there are any aborted memory regions.
   2.103 +  bool has_aborted_regions();
   2.104 +
   2.105    bool concurrent_marking_in_progress() {
   2.106      return _concurrent_marking_in_progress;
   2.107    }
   2.108 @@ -856,6 +877,15 @@
   2.109    // stack.
   2.110    HeapWord*                   _region_finger;
   2.111  
   2.112 +  // If we abort while scanning a region we record the remaining
   2.113 +  // unscanned portion and check this field when marking restarts.
   2.114 +  // This avoids having to push on the region stack while other
   2.115 +  // marking threads may still be popping regions.
   2.116 +  // If we were to push the unscanned portion directly to the
   2.117 +  // region stack then we would need to using locking versions
   2.118 +  // of the push and pop operations.
   2.119 +  MemRegion                   _aborted_region;
   2.120 +
   2.121    // the number of words this task has scanned
   2.122    size_t                      _words_scanned;
   2.123    // When _words_scanned reaches this limit, the regular clock is
   2.124 @@ -1012,6 +1042,15 @@
   2.125    void clear_has_aborted()      { _has_aborted = false; }
   2.126    bool claimed() { return _claimed; }
   2.127  
   2.128 +  // Support routines for the partially scanned region that may be
   2.129 +  // recorded as a result of aborting while draining the CMRegionStack
   2.130 +  MemRegion aborted_region()    { return _aborted_region; }
   2.131 +  void set_aborted_region(MemRegion mr)
   2.132 +                                { _aborted_region = mr; }
   2.133 +
   2.134 +  // Clears any recorded partially scanned region
   2.135 +  void clear_aborted_region()   { set_aborted_region(MemRegion()); }
   2.136 +
   2.137    void set_oop_closure(OopClosure* oop_closure) {
   2.138      _oop_closure = oop_closure;
   2.139    }