ysr@777: /*
tonyp@3114:  * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
ysr@777:  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
ysr@777:  *
ysr@777:  * This code is free software; you can redistribute it and/or modify it
ysr@777:  * under the terms of the GNU General Public License version 2 only, as
ysr@777:  * published by the Free Software Foundation.
ysr@777:  *
ysr@777:  * This code is distributed in the hope that it will be useful, but WITHOUT
ysr@777:  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
ysr@777:  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
ysr@777:  * version 2 for more details (a copy is included in the LICENSE file that
ysr@777:  * accompanied this code).
ysr@777:  *
ysr@777:  * You should have received a copy of the GNU General Public License version
ysr@777:  * 2 along with this work; if not, write to the Free Software Foundation,
ysr@777:  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
ysr@777:  *
trims@1907:  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
trims@1907:  * or visit www.oracle.com if you need additional information or have any
trims@1907:  * questions.
ysr@777:  *
ysr@777:  */
ysr@777: 
stefank@2314: #include "precompiled.hpp"
stefank@2314: #include "gc_implementation/g1/collectionSetChooser.hpp"
stefank@2314: #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
stefank@2314: #include "gc_implementation/g1/g1CollectorPolicy.hpp"
tonyp@3114: #include "gc_implementation/g1/g1ErgoVerbose.hpp"
stefank@2314: #include "memory/space.inline.hpp"
ysr@777: 
ysr@777: CSetChooserCache::CSetChooserCache() {
ysr@777:   for (int i = 0; i < CacheLength; ++i)
ysr@777:     _cache[i] = NULL;
ysr@777:   clear();
ysr@777: }
ysr@777: 
ysr@777: void CSetChooserCache::clear() {
ysr@777:   _occupancy = 0;
ysr@777:   _first = 0;
ysr@777:   for (int i = 0; i < CacheLength; ++i) {
ysr@777:     HeapRegion *hr = _cache[i];
ysr@777:     if (hr != NULL)
ysr@777:       hr->set_sort_index(-1);
ysr@777:     _cache[i] = NULL;
ysr@777:   }
ysr@777: }
ysr@777: 
ysr@777: #ifndef PRODUCT
ysr@777: bool CSetChooserCache::verify() {
ysr@777:   int index = _first;
ysr@777:   HeapRegion *prev = NULL;
ysr@777:   for (int i = 0; i < _occupancy; ++i) {
ysr@777:     guarantee(_cache[index] != NULL, "cache entry should not be empty");
ysr@777:     HeapRegion *hr = _cache[index];
ysr@777:     guarantee(!hr->is_young(), "should not be young!");
ysr@777:     if (prev != NULL) {
ysr@777:       guarantee(prev->gc_efficiency() >= hr->gc_efficiency(),
ysr@777:                 "cache should be correctly ordered");
ysr@777:     }
ysr@777:     guarantee(hr->sort_index() == get_sort_index(index),
ysr@777:               "sort index should be correct");
ysr@777:     index = trim_index(index + 1);
ysr@777:     prev = hr;
ysr@777:   }
ysr@777: 
ysr@777:   for (int i = 0; i < (CacheLength - _occupancy); ++i) {
ysr@777:     guarantee(_cache[index] == NULL, "cache entry should be empty");
ysr@777:     index = trim_index(index + 1);
ysr@777:   }
ysr@777: 
ysr@777:   guarantee(index == _first, "we should have reached where we started from");
ysr@777:   return true;
ysr@777: }
ysr@777: #endif // PRODUCT
ysr@777: 
ysr@777: void CSetChooserCache::insert(HeapRegion *hr) {
ysr@777:   assert(!is_full(), "cache should not be empty");
ysr@777:   hr->calc_gc_efficiency();
ysr@777: 
ysr@777:   int empty_index;
ysr@777:   if (_occupancy == 0) {
ysr@777:     empty_index = _first;
ysr@777:   } else {
ysr@777:     empty_index = trim_index(_first + _occupancy);
ysr@777:     assert(_cache[empty_index] == NULL, "last slot should be empty");
ysr@777:     int last_index = trim_index(empty_index - 1);
ysr@777:     HeapRegion *last = _cache[last_index];
ysr@777:     assert(last != NULL,"as the cache is not empty, last should not be empty");
ysr@777:     while (empty_index != _first &&
ysr@777:            last->gc_efficiency() < hr->gc_efficiency()) {
ysr@777:       _cache[empty_index] = last;
ysr@777:       last->set_sort_index(get_sort_index(empty_index));
ysr@777:       empty_index = last_index;
ysr@777:       last_index = trim_index(last_index - 1);
ysr@777:       last = _cache[last_index];
ysr@777:     }
ysr@777:   }
ysr@777:   _cache[empty_index] = hr;
ysr@777:   hr->set_sort_index(get_sort_index(empty_index));
ysr@777: 
ysr@777:   ++_occupancy;
ysr@777:   assert(verify(), "cache should be consistent");
ysr@777: }
ysr@777: 
ysr@777: HeapRegion *CSetChooserCache::remove_first() {
ysr@777:   if (_occupancy > 0) {
ysr@777:     assert(_cache[_first] != NULL, "cache should have at least one region");
ysr@777:     HeapRegion *ret = _cache[_first];
ysr@777:     _cache[_first] = NULL;
ysr@777:     ret->set_sort_index(-1);
ysr@777:     --_occupancy;
ysr@777:     _first = trim_index(_first + 1);
ysr@777:     assert(verify(), "cache should be consistent");
ysr@777:     return ret;
ysr@777:   } else {
ysr@777:     return NULL;
ysr@777:   }
ysr@777: }
ysr@777: 
ysr@777: static inline int orderRegions(HeapRegion* hr1, HeapRegion* hr2) {
ysr@777:   if (hr1 == NULL) {
ysr@777:     if (hr2 == NULL) return 0;
ysr@777:     else return 1;
ysr@777:   } else if (hr2 == NULL) {
ysr@777:     return -1;
ysr@777:   }
ysr@777:   if (hr2->gc_efficiency() < hr1->gc_efficiency()) return -1;
ysr@777:   else if (hr1->gc_efficiency() < hr2->gc_efficiency()) return 1;
ysr@777:   else return 0;
ysr@777: }
ysr@777: 
ysr@777: static int orderRegions(HeapRegion** hr1p, HeapRegion** hr2p) {
ysr@777:   return orderRegions(*hr1p, *hr2p);
ysr@777: }
ysr@777: 
ysr@777: CollectionSetChooser::CollectionSetChooser() :
ysr@777:   // The line below is the worst bit of C++ hackery I've ever written
ysr@777:   // (Detlefs, 11/23).  You should think of it as equivalent to
ysr@777:   // "_regions(100, true)": initialize the growable array and inform it
kvn@2043:   // that it should allocate its elem array(s) on the C heap.
kvn@2043:   //
kvn@2043:   // The first argument, however, is actually a comma expression
kvn@2043:   // (set_allocation_type(this, C_HEAP), 100). The purpose of the
kvn@2043:   // set_allocation_type() call is to replace the default allocation
kvn@2043:   // type for embedded objects STACK_OR_EMBEDDED with C_HEAP. It will
kvn@2043:   // allow to pass the assert in GenericGrowableArray() which checks
kvn@2043:   // that a growable array object must be on C heap if elements are.
kvn@2043:   //
kvn@2043:   // Note: containing object is allocated on C heap since it is CHeapObj.
kvn@2043:   //
kvn@2043:   _markedRegions((ResourceObj::set_allocation_type((address)&_markedRegions,
ysr@777:                                              ResourceObj::C_HEAP),
ysr@777:                   100),
ysr@777:                  true),
ysr@777:   _curMarkedIndex(0),
ysr@777:   _numMarkedRegions(0),
ysr@777:   _unmarked_age_1_returned_as_new(false),
ysr@777:   _first_par_unreserved_idx(0)
ysr@777: {}
ysr@777: 
ysr@777: 
ysr@777: 
ysr@777: #ifndef PRODUCT
ysr@777: bool CollectionSetChooser::verify() {
ysr@777:   int index = 0;
ysr@777:   guarantee(_curMarkedIndex <= _numMarkedRegions,
ysr@777:             "_curMarkedIndex should be within bounds");
ysr@777:   while (index < _curMarkedIndex) {
ysr@777:     guarantee(_markedRegions.at(index++) == NULL,
ysr@777:               "all entries before _curMarkedIndex should be NULL");
ysr@777:   }
ysr@777:   HeapRegion *prev = NULL;
ysr@777:   while (index < _numMarkedRegions) {
ysr@777:     HeapRegion *curr = _markedRegions.at(index++);
ysr@3185:     guarantee(curr != NULL, "Regions in _markedRegions array cannot be NULL");
ysr@3185:     int si = curr->sort_index();
ysr@3185:     guarantee(!curr->is_young(), "should not be young!");
ysr@3185:     guarantee(si > -1 && si == (index-1), "sort index invariant");
ysr@3185:     if (prev != NULL) {
ysr@3185:       guarantee(orderRegions(prev, curr) != 1, "regions should be sorted");
ysr@777:     }
ysr@3185:     prev = curr;
ysr@777:   }
ysr@777:   return _cache.verify();
ysr@777: }
ysr@777: #endif
ysr@777: 
ysr@777: void
ysr@777: CollectionSetChooser::fillCache() {
ysr@3185:   while (!_cache.is_full() && (_curMarkedIndex < _numMarkedRegions)) {
ysr@3185:     HeapRegion* hr = _markedRegions.at(_curMarkedIndex);
ysr@3185:     assert(hr != NULL,
ysr@3185:            err_msg("Unexpected NULL hr in _markedRegions at index %d",
ysr@3185:                    _curMarkedIndex));
ysr@3185:     _curMarkedIndex += 1;
ysr@3185:     assert(!hr->is_young(), "should not be young!");
ysr@3185:     assert(hr->sort_index() == _curMarkedIndex-1, "sort_index invariant");
ysr@3185:     _markedRegions.at_put(hr->sort_index(), NULL);
ysr@3185:     _cache.insert(hr);
ysr@3185:     assert(!_cache.is_empty(), "cache should not be empty");
ysr@777:   }
ysr@3185:   assert(verify(), "cache should be consistent");
ysr@777: }
ysr@777: 
ysr@777: void
ysr@777: CollectionSetChooser::sortMarkedHeapRegions() {
ysr@777:   guarantee(_cache.is_empty(), "cache should be empty");
ysr@777:   // First trim any unused portion of the top in the parallel case.
ysr@777:   if (_first_par_unreserved_idx > 0) {
ysr@777:     if (G1PrintParCleanupStats) {
ysr@777:       gclog_or_tty->print("     Truncating _markedRegions from %d to %d.\n",
ysr@777:                           _markedRegions.length(), _first_par_unreserved_idx);
ysr@777:     }
ysr@777:     assert(_first_par_unreserved_idx <= _markedRegions.length(),
ysr@777:            "Or we didn't reserved enough length");
ysr@777:     _markedRegions.trunc_to(_first_par_unreserved_idx);
ysr@777:   }
ysr@777:   _markedRegions.sort(orderRegions);
ysr@777:   assert(_numMarkedRegions <= _markedRegions.length(), "Requirement");
ysr@777:   assert(_numMarkedRegions == 0
ysr@777:          || _markedRegions.at(_numMarkedRegions-1) != NULL,
ysr@777:          "Testing _numMarkedRegions");
ysr@777:   assert(_numMarkedRegions == _markedRegions.length()
ysr@777:          || _markedRegions.at(_numMarkedRegions) == NULL,
ysr@777:          "Testing _numMarkedRegions");
ysr@777:   if (G1PrintParCleanupStats) {
ysr@777:     gclog_or_tty->print_cr("     Sorted %d marked regions.", _numMarkedRegions);
ysr@777:   }
ysr@777:   for (int i = 0; i < _numMarkedRegions; i++) {
ysr@777:     assert(_markedRegions.at(i) != NULL, "Should be true by sorting!");
ysr@777:     _markedRegions.at(i)->set_sort_index(i);
tonyp@2717:   }
tonyp@2717:   if (G1PrintRegionLivenessInfo) {
tonyp@2717:     G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Sorting");
tonyp@2717:     for (int i = 0; i < _numMarkedRegions; ++i) {
tonyp@2717:       HeapRegion* r = _markedRegions.at(i);
tonyp@2717:       cl.doHeapRegion(r);
ysr@777:     }
ysr@777:   }
ysr@777:   assert(verify(), "should now be sorted");
ysr@777: }
ysr@777: 
ysr@777: void
ysr@777: CollectionSetChooser::addMarkedHeapRegion(HeapRegion* hr) {
ysr@777:   assert(!hr->isHumongous(),
ysr@777:          "Humongous regions shouldn't be added to the collection set");
ysr@777:   assert(!hr->is_young(), "should not be young!");
ysr@777:   _markedRegions.append(hr);
ysr@777:   _numMarkedRegions++;
ysr@777:   hr->calc_gc_efficiency();
ysr@777: }
ysr@777: 
ysr@777: void
ysr@777: CollectionSetChooser::
ysr@777: prepareForAddMarkedHeapRegionsPar(size_t n_regions, size_t chunkSize) {
ysr@777:   _first_par_unreserved_idx = 0;
jmasa@3294:   int n_threads = ParallelGCThreads;
jmasa@3294:   if (UseDynamicNumberOfGCThreads) {
jmasa@3294:     assert(G1CollectedHeap::heap()->workers()->active_workers() > 0,
jmasa@3294:       "Should have been set earlier");
jmasa@3294:     // This is defensive code. As the assertion above says, the number
jmasa@3294:     // of active threads should be > 0, but in case there is some path
jmasa@3294:     // or some improperly initialized variable with leads to no
jmasa@3294:     // active threads, protect against that in a product build.
jmasa@3294:     n_threads = MAX2(G1CollectedHeap::heap()->workers()->active_workers(),
jmasa@3357:                      1U);
jmasa@3294:   }
jmasa@3294:   size_t max_waste = n_threads * chunkSize;
ysr@777:   // it should be aligned with respect to chunkSize
ysr@777:   size_t aligned_n_regions =
ysr@777:                      (n_regions + (chunkSize - 1)) / chunkSize * chunkSize;
ysr@777:   assert( aligned_n_regions % chunkSize == 0, "should be aligned" );
ysr@777:   _markedRegions.at_put_grow((int)(aligned_n_regions + max_waste - 1), NULL);
ysr@777: }
ysr@777: 
ysr@777: jint
ysr@777: CollectionSetChooser::getParMarkedHeapRegionChunk(jint n_regions) {
jmasa@3294:   // Don't do this assert because this can be called at a point
jmasa@3294:   // where the loop up stream will not execute again but might
jmasa@3294:   // try to claim more chunks (loop test has not been done yet).
jmasa@3294:   // assert(_markedRegions.length() > _first_par_unreserved_idx,
jmasa@3294:   //  "Striding beyond the marked regions");
ysr@777:   jint res = Atomic::add(n_regions, &_first_par_unreserved_idx);
ysr@777:   assert(_markedRegions.length() > res + n_regions - 1,
ysr@777:          "Should already have been expanded");
ysr@777:   return res - n_regions;
ysr@777: }
ysr@777: 
ysr@777: void
ysr@777: CollectionSetChooser::setMarkedHeapRegion(jint index, HeapRegion* hr) {
ysr@777:   assert(_markedRegions.at(index) == NULL, "precondition");
ysr@777:   assert(!hr->is_young(), "should not be young!");
ysr@777:   _markedRegions.at_put(index, hr);
ysr@777:   hr->calc_gc_efficiency();
ysr@777: }
ysr@777: 
ysr@777: void
ysr@777: CollectionSetChooser::incNumMarkedHeapRegions(jint inc_by) {
ysr@777:   (void)Atomic::add(inc_by, &_numMarkedRegions);
ysr@777: }
ysr@777: 
ysr@777: void
ysr@777: CollectionSetChooser::clearMarkedHeapRegions(){
ysr@777:   for (int i = 0; i < _markedRegions.length(); i++) {
ysr@777:     HeapRegion* r =   _markedRegions.at(i);
ysr@777:     if (r != NULL) r->set_sort_index(-1);
ysr@777:   }
ysr@777:   _markedRegions.clear();
ysr@777:   _curMarkedIndex = 0;
ysr@777:   _numMarkedRegions = 0;
ysr@777:   _cache.clear();
ysr@777: };
ysr@777: 
ysr@777: void
ysr@777: CollectionSetChooser::updateAfterFullCollection() {
ysr@777:   clearMarkedHeapRegions();
ysr@777: }
ysr@777: 
ysr@777: // if time_remaining < 0.0, then this method should try to return
ysr@777: // a region, whether it fits within the remaining time or not
ysr@777: HeapRegion*
ysr@777: CollectionSetChooser::getNextMarkedRegion(double time_remaining,
ysr@777:                                           double avg_prediction) {
ysr@777:   G1CollectedHeap* g1h = G1CollectedHeap::heap();
ysr@777:   G1CollectorPolicy* g1p = g1h->g1_policy();
ysr@777:   fillCache();
ysr@777:   if (_cache.is_empty()) {
ysr@777:     assert(_curMarkedIndex == _numMarkedRegions,
ysr@777:            "if cache is empty, list should also be empty");
tonyp@3114:     ergo_verbose0(ErgoCSetConstruction,
tonyp@3114:                   "stop adding old regions to CSet",
tonyp@3114:                   ergo_format_reason("cache is empty"));
ysr@777:     return NULL;
ysr@777:   }
ysr@777: 
ysr@777:   HeapRegion *hr = _cache.get_first();
ysr@777:   assert(hr != NULL, "if cache not empty, first entry should be non-null");
ysr@777:   double predicted_time = g1h->predict_region_elapsed_time_ms(hr, false);
ysr@777: 
ysr@777:   if (g1p->adaptive_young_list_length()) {
ysr@777:     if (time_remaining - predicted_time < 0.0) {
ysr@777:       g1h->check_if_region_is_too_expensive(predicted_time);
tonyp@3114:       ergo_verbose2(ErgoCSetConstruction,
tonyp@3114:                     "stop adding old regions to CSet",
tonyp@3114:                     ergo_format_reason("predicted old region time higher than remaining time")
tonyp@3114:                     ergo_format_ms("predicted old region time")
tonyp@3114:                     ergo_format_ms("remaining time"),
tonyp@3114:                     predicted_time, time_remaining);
ysr@777:       return NULL;
ysr@777:     }
ysr@777:   } else {
tonyp@3114:     double threshold = 2.0 * avg_prediction;
tonyp@3114:     if (predicted_time > threshold) {
tonyp@3114:       ergo_verbose2(ErgoCSetConstruction,
tonyp@3114:                     "stop adding old regions to CSet",
tonyp@3114:                     ergo_format_reason("predicted old region time higher than threshold")
tonyp@3114:                     ergo_format_ms("predicted old region time")
tonyp@3114:                     ergo_format_ms("threshold"),
tonyp@3114:                     predicted_time, threshold);
ysr@777:       return NULL;
ysr@777:     }
ysr@777:   }
ysr@777: 
ysr@777:   HeapRegion *hr2 = _cache.remove_first();
ysr@777:   assert(hr == hr2, "cache contents should not have changed");
ysr@777: 
ysr@777:   return hr;
ysr@777: }