1.1 --- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Mon Apr 19 05:40:21 2010 -0700
1.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Thu Apr 22 10:02:38 2010 -0700
1.3 @@ -1,5 +1,5 @@
1.4 /*
1.5 - * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved.
1.6 + * Copyright 2001-2010 Sun Microsystems, Inc. All Rights Reserved.
1.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
1.8 *
1.9 * This code is free software; you can redistribute it and/or modify it
1.10 @@ -42,10 +42,6 @@
1.11 0.01, 0.005, 0.005, 0.003, 0.003, 0.002, 0.002, 0.0015
1.12 };
1.13
1.14 -static double cost_per_scan_only_region_ms_defaults[] = {
1.15 - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0
1.16 -};
1.17 -
1.18 // all the same
1.19 static double fully_young_cards_per_entry_ratio_defaults[] = {
1.20 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0
1.21 @@ -125,7 +121,6 @@
1.22 _pending_card_diff_seq(new TruncatedSeq(TruncatedSeqLength)),
1.23 _rs_length_diff_seq(new TruncatedSeq(TruncatedSeqLength)),
1.24 _cost_per_card_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
1.25 - _cost_per_scan_only_region_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
1.26 _fully_young_cards_per_entry_ratio_seq(new TruncatedSeq(TruncatedSeqLength)),
1.27 _partially_young_cards_per_entry_ratio_seq(
1.28 new TruncatedSeq(TruncatedSeqLength)),
1.29 @@ -133,7 +128,6 @@
1.30 _partially_young_cost_per_entry_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
1.31 _cost_per_byte_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
1.32 _cost_per_byte_ms_during_cm_seq(new TruncatedSeq(TruncatedSeqLength)),
1.33 - _cost_per_scan_only_region_ms_during_cm_seq(new TruncatedSeq(TruncatedSeqLength)),
1.34 _constant_other_time_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
1.35 _young_other_cost_per_region_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
1.36 _non_young_other_cost_per_region_ms_seq(
1.37 @@ -186,6 +180,22 @@
1.38 _prev_collection_pause_used_at_end_bytes(0),
1.39
1.40 _collection_set(NULL),
1.41 + _collection_set_size(0),
1.42 + _collection_set_bytes_used_before(0),
1.43 +
1.44 + // Incremental CSet attributes
1.45 + _inc_cset_build_state(Inactive),
1.46 + _inc_cset_head(NULL),
1.47 + _inc_cset_tail(NULL),
1.48 + _inc_cset_size(0),
1.49 + _inc_cset_young_index(0),
1.50 + _inc_cset_bytes_used_before(0),
1.51 + _inc_cset_max_finger(NULL),
1.52 + _inc_cset_recorded_young_bytes(0),
1.53 + _inc_cset_recorded_rs_lengths(0),
1.54 + _inc_cset_predicted_elapsed_time_ms(0.0),
1.55 + _inc_cset_predicted_bytes_to_copy(0),
1.56 +
1.57 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
1.58 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
1.59 #endif // _MSC_VER
1.60 @@ -223,8 +233,6 @@
1.61
1.62 _par_last_ext_root_scan_times_ms = new double[_parallel_gc_threads];
1.63 _par_last_mark_stack_scan_times_ms = new double[_parallel_gc_threads];
1.64 - _par_last_scan_only_times_ms = new double[_parallel_gc_threads];
1.65 - _par_last_scan_only_regions_scanned = new double[_parallel_gc_threads];
1.66
1.67 _par_last_update_rs_start_times_ms = new double[_parallel_gc_threads];
1.68 _par_last_update_rs_times_ms = new double[_parallel_gc_threads];
1.69 @@ -254,8 +262,6 @@
1.70 _pending_card_diff_seq->add(0.0);
1.71 _rs_length_diff_seq->add(rs_length_diff_defaults[index]);
1.72 _cost_per_card_ms_seq->add(cost_per_card_ms_defaults[index]);
1.73 - _cost_per_scan_only_region_ms_seq->add(
1.74 - cost_per_scan_only_region_ms_defaults[index]);
1.75 _fully_young_cards_per_entry_ratio_seq->add(
1.76 fully_young_cards_per_entry_ratio_defaults[index]);
1.77 _cost_per_entry_ms_seq->add(cost_per_entry_ms_defaults[index]);
1.78 @@ -283,7 +289,7 @@
1.79
1.80 // if G1FixedSurvivorSpaceSize is 0 which means the size is not
1.81 // fixed, then _max_survivor_regions will be calculated at
1.82 - // calculate_young_list_target_config during initialization
1.83 + // calculate_young_list_target_length during initialization
1.84 _max_survivor_regions = G1FixedSurvivorSpaceSize / HeapRegion::GrainBytes;
1.85
1.86 assert(GCTimeRatio > 0,
1.87 @@ -357,15 +363,18 @@
1.88 set_adaptive_young_list_length(false);
1.89 _young_list_fixed_length = initial_region_num;
1.90 }
1.91 - _free_regions_at_end_of_collection = _g1->free_regions();
1.92 - _scan_only_regions_at_end_of_collection = 0;
1.93 - calculate_young_list_min_length();
1.94 - guarantee( _young_list_min_length == 0, "invariant, not enough info" );
1.95 - calculate_young_list_target_config();
1.96 - } else {
1.97 + _free_regions_at_end_of_collection = _g1->free_regions();
1.98 + calculate_young_list_min_length();
1.99 + guarantee( _young_list_min_length == 0, "invariant, not enough info" );
1.100 + calculate_young_list_target_length();
1.101 + } else {
1.102 _young_list_fixed_length = 0;
1.103 _in_young_gc_mode = false;
1.104 }
1.105 +
1.106 + // We may immediately start allocating regions and placing them on the
1.107 + // collection set list. Initialize the per-collection set info
1.108 + start_incremental_cset_building();
1.109 }
1.110
1.111 // Create the jstat counters for the policy.
1.112 @@ -385,112 +394,29 @@
1.113 double when_ms = _mmu_tracker->when_max_gc_sec(now_sec) * 1000.0;
1.114 double alloc_rate_ms = predict_alloc_rate_ms();
1.115 int min_regions = (int) ceil(alloc_rate_ms * when_ms);
1.116 - int current_region_num = (int) _g1->young_list_length();
1.117 + int current_region_num = (int) _g1->young_list()->length();
1.118 _young_list_min_length = min_regions + current_region_num;
1.119 }
1.120 }
1.121
1.122 -void G1CollectorPolicy::calculate_young_list_target_config() {
1.123 +void G1CollectorPolicy::calculate_young_list_target_length() {
1.124 if (adaptive_young_list_length()) {
1.125 size_t rs_lengths = (size_t) get_new_prediction(_rs_lengths_seq);
1.126 - calculate_young_list_target_config(rs_lengths);
1.127 + calculate_young_list_target_length(rs_lengths);
1.128 } else {
1.129 if (full_young_gcs())
1.130 _young_list_target_length = _young_list_fixed_length;
1.131 else
1.132 _young_list_target_length = _young_list_fixed_length / 2;
1.133 +
1.134 _young_list_target_length = MAX2(_young_list_target_length, (size_t)1);
1.135 - size_t so_length = calculate_optimal_so_length(_young_list_target_length);
1.136 - guarantee( so_length < _young_list_target_length, "invariant" );
1.137 - _young_list_so_prefix_length = so_length;
1.138 }
1.139 calculate_survivors_policy();
1.140 }
1.141
1.142 -// This method calculate the optimal scan-only set for a fixed young
1.143 -// gen size. I couldn't work out how to reuse the more elaborate one,
1.144 -// i.e. calculate_young_list_target_config(rs_length), as the loops are
1.145 -// fundamentally different (the other one finds a config for different
1.146 -// S-O lengths, whereas here we need to do the opposite).
1.147 -size_t G1CollectorPolicy::calculate_optimal_so_length(
1.148 - size_t young_list_length) {
1.149 - if (!G1UseScanOnlyPrefix)
1.150 - return 0;
1.151 -
1.152 - if (_all_pause_times_ms->num() < 3) {
1.153 - // we won't use a scan-only set at the beginning to allow the rest
1.154 - // of the predictors to warm up
1.155 - return 0;
1.156 - }
1.157 -
1.158 - if (_cost_per_scan_only_region_ms_seq->num() < 3) {
1.159 - // then, we'll only set the S-O set to 1 for a little bit of time,
1.160 - // to get enough information on the scanning cost
1.161 - return 1;
1.162 - }
1.163 -
1.164 - size_t pending_cards = (size_t) get_new_prediction(_pending_cards_seq);
1.165 - size_t rs_lengths = (size_t) get_new_prediction(_rs_lengths_seq);
1.166 - size_t adj_rs_lengths = rs_lengths + predict_rs_length_diff();
1.167 - size_t scanned_cards;
1.168 - if (full_young_gcs())
1.169 - scanned_cards = predict_young_card_num(adj_rs_lengths);
1.170 - else
1.171 - scanned_cards = predict_non_young_card_num(adj_rs_lengths);
1.172 - double base_time_ms = predict_base_elapsed_time_ms(pending_cards,
1.173 - scanned_cards);
1.174 -
1.175 - size_t so_length = 0;
1.176 - double max_gc_eff = 0.0;
1.177 - for (size_t i = 0; i < young_list_length; ++i) {
1.178 - double gc_eff = 0.0;
1.179 - double pause_time_ms = 0.0;
1.180 - predict_gc_eff(young_list_length, i, base_time_ms,
1.181 - &gc_eff, &pause_time_ms);
1.182 - if (gc_eff > max_gc_eff) {
1.183 - max_gc_eff = gc_eff;
1.184 - so_length = i;
1.185 - }
1.186 - }
1.187 -
1.188 - // set it to 95% of the optimal to make sure we sample the "area"
1.189 - // around the optimal length to get up-to-date survival rate data
1.190 - return so_length * 950 / 1000;
1.191 -}
1.192 -
1.193 -// This is a really cool piece of code! It finds the best
1.194 -// target configuration (young length / scan-only prefix length) so
1.195 -// that GC efficiency is maximized and that we also meet a pause
1.196 -// time. It's a triple nested loop. These loops are explained below
1.197 -// from the inside-out :-)
1.198 -//
1.199 -// (a) The innermost loop will try to find the optimal young length
1.200 -// for a fixed S-O length. It uses a binary search to speed up the
1.201 -// process. We assume that, for a fixed S-O length, as we add more
1.202 -// young regions to the CSet, the GC efficiency will only go up (I'll
1.203 -// skip the proof). So, using a binary search to optimize this process
1.204 -// makes perfect sense.
1.205 -//
1.206 -// (b) The middle loop will fix the S-O length before calling the
1.207 -// innermost one. It will vary it between two parameters, increasing
1.208 -// it by a given increment.
1.209 -//
1.210 -// (c) The outermost loop will call the middle loop three times.
1.211 -// (1) The first time it will explore all possible S-O length values
1.212 -// from 0 to as large as it can get, using a coarse increment (to
1.213 -// quickly "home in" to where the optimal seems to be).
1.214 -// (2) The second time it will explore the values around the optimal
1.215 -// that was found by the first iteration using a fine increment.
1.216 -// (3) Once the optimal config has been determined by the second
1.217 -// iteration, we'll redo the calculation, but setting the S-O length
1.218 -// to 95% of the optimal to make sure we sample the "area"
1.219 -// around the optimal length to get up-to-date survival rate data
1.220 -//
1.221 -// Termination conditions for the iterations are several: the pause
1.222 -// time is over the limit, we do not have enough to-space, etc.
1.223 -
1.224 -void G1CollectorPolicy::calculate_young_list_target_config(size_t rs_lengths) {
1.225 +void G1CollectorPolicy::calculate_young_list_target_length(size_t rs_lengths) {
1.226 guarantee( adaptive_young_list_length(), "pre-condition" );
1.227 + guarantee( !_in_marking_window || !_last_full_young_gc, "invariant" );
1.228
1.229 double start_time_sec = os::elapsedTime();
1.230 size_t min_reserve_perc = MAX2((size_t)2, (size_t)G1ReservePercent);
1.231 @@ -504,285 +430,80 @@
1.232 double survivor_regions_evac_time =
1.233 predict_survivor_regions_evac_time();
1.234
1.235 - size_t min_so_length = 0;
1.236 - size_t max_so_length = 0;
1.237 -
1.238 - if (G1UseScanOnlyPrefix) {
1.239 - if (_all_pause_times_ms->num() < 3) {
1.240 - // we won't use a scan-only set at the beginning to allow the rest
1.241 - // of the predictors to warm up
1.242 - min_so_length = 0;
1.243 - max_so_length = 0;
1.244 - } else if (_cost_per_scan_only_region_ms_seq->num() < 3) {
1.245 - // then, we'll only set the S-O set to 1 for a little bit of time,
1.246 - // to get enough information on the scanning cost
1.247 - min_so_length = 1;
1.248 - max_so_length = 1;
1.249 - } else if (_in_marking_window || _last_full_young_gc) {
1.250 - // no S-O prefix during a marking phase either, as at the end
1.251 - // of the marking phase we'll have to use a very small young
1.252 - // length target to fill up the rest of the CSet with
1.253 - // non-young regions and, if we have lots of scan-only regions
1.254 - // left-over, we will not be able to add any more non-young
1.255 - // regions.
1.256 - min_so_length = 0;
1.257 - max_so_length = 0;
1.258 - } else {
1.259 - // this is the common case; we'll never reach the maximum, we
1.260 - // one of the end conditions will fire well before that
1.261 - // (hopefully!)
1.262 - min_so_length = 0;
1.263 - max_so_length = _free_regions_at_end_of_collection - 1;
1.264 - }
1.265 - } else {
1.266 - // no S-O prefix, as the switch is not set, but we still need to
1.267 - // do one iteration to calculate the best young target that
1.268 - // meets the pause time; this way we reuse the same code instead
1.269 - // of replicating it
1.270 - min_so_length = 0;
1.271 - max_so_length = 0;
1.272 - }
1.273 -
1.274 double target_pause_time_ms = _mmu_tracker->max_gc_time() * 1000.0;
1.275 size_t pending_cards = (size_t) get_new_prediction(_pending_cards_seq);
1.276 size_t adj_rs_lengths = rs_lengths + predict_rs_length_diff();
1.277 - size_t scanned_cards;
1.278 - if (full_young_gcs())
1.279 - scanned_cards = predict_young_card_num(adj_rs_lengths);
1.280 - else
1.281 - scanned_cards = predict_non_young_card_num(adj_rs_lengths);
1.282 - // calculate this once, so that we don't have to recalculate it in
1.283 - // the innermost loop
1.284 + size_t scanned_cards = predict_young_card_num(adj_rs_lengths);
1.285 double base_time_ms = predict_base_elapsed_time_ms(pending_cards, scanned_cards)
1.286 + survivor_regions_evac_time;
1.287 +
1.288 // the result
1.289 size_t final_young_length = 0;
1.290 - size_t final_so_length = 0;
1.291 - double final_gc_eff = 0.0;
1.292 - // we'll also keep track of how many times we go into the inner loop
1.293 - // this is for profiling reasons
1.294 - size_t calculations = 0;
1.295 -
1.296 - // this determines which of the three iterations the outer loop is in
1.297 - typedef enum {
1.298 - pass_type_coarse,
1.299 - pass_type_fine,
1.300 - pass_type_final
1.301 - } pass_type_t;
1.302 -
1.303 - // range of the outer loop's iteration
1.304 - size_t from_so_length = min_so_length;
1.305 - size_t to_so_length = max_so_length;
1.306 - guarantee( from_so_length <= to_so_length, "invariant" );
1.307 -
1.308 - // this will keep the S-O length that's found by the second
1.309 - // iteration of the outer loop; we'll keep it just in case the third
1.310 - // iteration fails to find something
1.311 - size_t fine_so_length = 0;
1.312 -
1.313 - // the increment step for the coarse (first) iteration
1.314 - size_t so_coarse_increments = 5;
1.315 -
1.316 - // the common case, we'll start with the coarse iteration
1.317 - pass_type_t pass = pass_type_coarse;
1.318 - size_t so_length_incr = so_coarse_increments;
1.319 -
1.320 - if (from_so_length == to_so_length) {
1.321 - // not point in doing the coarse iteration, we'll go directly into
1.322 - // the fine one (we essentially trying to find the optimal young
1.323 - // length for a fixed S-O length).
1.324 - so_length_incr = 1;
1.325 - pass = pass_type_final;
1.326 - } else if (to_so_length - from_so_length < 3 * so_coarse_increments) {
1.327 - // again, the range is too short so no point in foind the coarse
1.328 - // iteration either
1.329 - so_length_incr = 1;
1.330 - pass = pass_type_fine;
1.331 +
1.332 + size_t init_free_regions =
1.333 + MAX2((size_t)0, _free_regions_at_end_of_collection - reserve_regions);
1.334 +
1.335 + // if we're still under the pause target...
1.336 + if (base_time_ms <= target_pause_time_ms) {
1.337 + // We make sure that the shortest young length that makes sense
1.338 + // fits within the target pause time.
1.339 + size_t min_young_length = 1;
1.340 +
1.341 + if (predict_will_fit(min_young_length, base_time_ms,
1.342 + init_free_regions, target_pause_time_ms)) {
1.343 + // The shortest young length will fit within the target pause time;
1.344 + // we'll now check whether the absolute maximum number of young
1.345 + // regions will fit in the target pause time. If not, we'll do
1.346 + // a binary search between min_young_length and max_young_length
1.347 + size_t abs_max_young_length = _free_regions_at_end_of_collection - 1;
1.348 + size_t max_young_length = abs_max_young_length;
1.349 +
1.350 + if (max_young_length > min_young_length) {
1.351 + // Let's check if the initial max young length will fit within the
1.352 + // target pause. If so then there is no need to search for a maximal
1.353 + // young length - we'll return the initial maximum
1.354 +
1.355 + if (predict_will_fit(max_young_length, base_time_ms,
1.356 + init_free_regions, target_pause_time_ms)) {
1.357 + // The maximum young length will satisfy the target pause time.
1.358 + // We are done so set min young length to this maximum length.
1.359 + // The code after the loop will then set final_young_length using
1.360 + // the value cached in the minimum length.
1.361 + min_young_length = max_young_length;
1.362 + } else {
1.363 + // The maximum possible number of young regions will not fit within
1.364 + // the target pause time so let's search....
1.365 +
1.366 + size_t diff = (max_young_length - min_young_length) / 2;
1.367 + max_young_length = min_young_length + diff;
1.368 +
1.369 + while (max_young_length > min_young_length) {
1.370 + if (predict_will_fit(max_young_length, base_time_ms,
1.371 + init_free_regions, target_pause_time_ms)) {
1.372 +
1.373 + // The current max young length will fit within the target
1.374 + // pause time. Note we do not exit the loop here. By setting
1.375 + // min = max, and then increasing the max below means that
1.376 + // we will continue searching for an upper bound in the
1.377 + // range [max..max+diff]
1.378 + min_young_length = max_young_length;
1.379 + }
1.380 + diff = (max_young_length - min_young_length) / 2;
1.381 + max_young_length = min_young_length + diff;
1.382 + }
1.383 + // the above loop found a maximal young length that will fit
1.384 + // within the target pause time.
1.385 + }
1.386 + assert(min_young_length <= abs_max_young_length, "just checking");
1.387 + }
1.388 + final_young_length = min_young_length;
1.389 + }
1.390 }
1.391 -
1.392 - bool done = false;
1.393 - // this is the outermost loop
1.394 - while (!done) {
1.395 -#ifdef TRACE_CALC_YOUNG_CONFIG
1.396 - // leave this in for debugging, just in case
1.397 - gclog_or_tty->print_cr("searching between " SIZE_FORMAT " and " SIZE_FORMAT
1.398 - ", incr " SIZE_FORMAT ", pass %s",
1.399 - from_so_length, to_so_length, so_length_incr,
1.400 - (pass == pass_type_coarse) ? "coarse" :
1.401 - (pass == pass_type_fine) ? "fine" : "final");
1.402 -#endif // TRACE_CALC_YOUNG_CONFIG
1.403 -
1.404 - size_t so_length = from_so_length;
1.405 - size_t init_free_regions =
1.406 - MAX2((size_t)0,
1.407 - _free_regions_at_end_of_collection +
1.408 - _scan_only_regions_at_end_of_collection - reserve_regions);
1.409 -
1.410 - // this determines whether a configuration was found
1.411 - bool gc_eff_set = false;
1.412 - // this is the middle loop
1.413 - while (so_length <= to_so_length) {
1.414 - // base time, which excludes region-related time; again we
1.415 - // calculate it once to avoid recalculating it in the
1.416 - // innermost loop
1.417 - double base_time_with_so_ms =
1.418 - base_time_ms + predict_scan_only_time_ms(so_length);
1.419 - // it's already over the pause target, go around
1.420 - if (base_time_with_so_ms > target_pause_time_ms)
1.421 - break;
1.422 -
1.423 - size_t starting_young_length = so_length+1;
1.424 -
1.425 - // we make sure that the short young length that makes sense
1.426 - // (one more than the S-O length) is feasible
1.427 - size_t min_young_length = starting_young_length;
1.428 - double min_gc_eff;
1.429 - bool min_ok;
1.430 - ++calculations;
1.431 - min_ok = predict_gc_eff(min_young_length, so_length,
1.432 - base_time_with_so_ms,
1.433 - init_free_regions, target_pause_time_ms,
1.434 - &min_gc_eff);
1.435 -
1.436 - if (min_ok) {
1.437 - // the shortest young length is indeed feasible; we'll know
1.438 - // set up the max young length and we'll do a binary search
1.439 - // between min_young_length and max_young_length
1.440 - size_t max_young_length = _free_regions_at_end_of_collection - 1;
1.441 - double max_gc_eff = 0.0;
1.442 - bool max_ok = false;
1.443 -
1.444 - // the innermost loop! (finally!)
1.445 - while (max_young_length > min_young_length) {
1.446 - // we'll make sure that min_young_length is always at a
1.447 - // feasible config
1.448 - guarantee( min_ok, "invariant" );
1.449 -
1.450 - ++calculations;
1.451 - max_ok = predict_gc_eff(max_young_length, so_length,
1.452 - base_time_with_so_ms,
1.453 - init_free_regions, target_pause_time_ms,
1.454 - &max_gc_eff);
1.455 -
1.456 - size_t diff = (max_young_length - min_young_length) / 2;
1.457 - if (max_ok) {
1.458 - min_young_length = max_young_length;
1.459 - min_gc_eff = max_gc_eff;
1.460 - min_ok = true;
1.461 - }
1.462 - max_young_length = min_young_length + diff;
1.463 - }
1.464 -
1.465 - // the innermost loop found a config
1.466 - guarantee( min_ok, "invariant" );
1.467 - if (min_gc_eff > final_gc_eff) {
1.468 - // it's the best config so far, so we'll keep it
1.469 - final_gc_eff = min_gc_eff;
1.470 - final_young_length = min_young_length;
1.471 - final_so_length = so_length;
1.472 - gc_eff_set = true;
1.473 - }
1.474 - }
1.475 -
1.476 - // incremental the fixed S-O length and go around
1.477 - so_length += so_length_incr;
1.478 - }
1.479 -
1.480 - // this is the end of the outermost loop and we need to decide
1.481 - // what to do during the next iteration
1.482 - if (pass == pass_type_coarse) {
1.483 - // we just did the coarse pass (first iteration)
1.484 -
1.485 - if (!gc_eff_set)
1.486 - // we didn't find a feasible config so we'll just bail out; of
1.487 - // course, it might be the case that we missed it; but I'd say
1.488 - // it's a bit unlikely
1.489 - done = true;
1.490 - else {
1.491 - // We did find a feasible config with optimal GC eff during
1.492 - // the first pass. So the second pass we'll only consider the
1.493 - // S-O lengths around that config with a fine increment.
1.494 -
1.495 - guarantee( so_length_incr == so_coarse_increments, "invariant" );
1.496 - guarantee( final_so_length >= min_so_length, "invariant" );
1.497 -
1.498 -#ifdef TRACE_CALC_YOUNG_CONFIG
1.499 - // leave this in for debugging, just in case
1.500 - gclog_or_tty->print_cr(" coarse pass: SO length " SIZE_FORMAT,
1.501 - final_so_length);
1.502 -#endif // TRACE_CALC_YOUNG_CONFIG
1.503 -
1.504 - from_so_length =
1.505 - (final_so_length - min_so_length > so_coarse_increments) ?
1.506 - final_so_length - so_coarse_increments + 1 : min_so_length;
1.507 - to_so_length =
1.508 - (max_so_length - final_so_length > so_coarse_increments) ?
1.509 - final_so_length + so_coarse_increments - 1 : max_so_length;
1.510 -
1.511 - pass = pass_type_fine;
1.512 - so_length_incr = 1;
1.513 - }
1.514 - } else if (pass == pass_type_fine) {
1.515 - // we just finished the second pass
1.516 -
1.517 - if (!gc_eff_set) {
1.518 - // we didn't find a feasible config (yes, it's possible;
1.519 - // notice that, sometimes, we go directly into the fine
1.520 - // iteration and skip the coarse one) so we bail out
1.521 - done = true;
1.522 - } else {
1.523 - // We did find a feasible config with optimal GC eff
1.524 - guarantee( so_length_incr == 1, "invariant" );
1.525 -
1.526 - if (final_so_length == 0) {
1.527 - // The config is of an empty S-O set, so we'll just bail out
1.528 - done = true;
1.529 - } else {
1.530 - // we'll go around once more, setting the S-O length to 95%
1.531 - // of the optimal
1.532 - size_t new_so_length = 950 * final_so_length / 1000;
1.533 -
1.534 -#ifdef TRACE_CALC_YOUNG_CONFIG
1.535 - // leave this in for debugging, just in case
1.536 - gclog_or_tty->print_cr(" fine pass: SO length " SIZE_FORMAT
1.537 - ", setting it to " SIZE_FORMAT,
1.538 - final_so_length, new_so_length);
1.539 -#endif // TRACE_CALC_YOUNG_CONFIG
1.540 -
1.541 - from_so_length = new_so_length;
1.542 - to_so_length = new_so_length;
1.543 - fine_so_length = final_so_length;
1.544 -
1.545 - pass = pass_type_final;
1.546 - }
1.547 - }
1.548 - } else if (pass == pass_type_final) {
1.549 - // we just finished the final (third) pass
1.550 -
1.551 - if (!gc_eff_set)
1.552 - // we didn't find a feasible config, so we'll just use the one
1.553 - // we found during the second pass, which we saved
1.554 - final_so_length = fine_so_length;
1.555 -
1.556 - // and we're done!
1.557 - done = true;
1.558 - } else {
1.559 - guarantee( false, "should never reach here" );
1.560 - }
1.561 -
1.562 - // we now go around the outermost loop
1.563 - }
1.564 + // and we're done!
1.565
1.566 // we should have at least one region in the target young length
1.567 _young_list_target_length =
1.568 MAX2((size_t) 1, final_young_length + _recorded_survivor_regions);
1.569 - if (final_so_length >= final_young_length)
1.570 - // and we need to ensure that the S-O length is not greater than
1.571 - // the target young length (this is being a bit careful)
1.572 - final_so_length = 0;
1.573 - _young_list_so_prefix_length = final_so_length;
1.574 - guarantee( !_in_marking_window || !_last_full_young_gc ||
1.575 - _young_list_so_prefix_length == 0, "invariant" );
1.576
1.577 // let's keep an eye of how long we spend on this calculation
1.578 // right now, I assume that we'll print it when we need it; we
1.579 @@ -790,142 +511,91 @@
1.580 double end_time_sec = os::elapsedTime();
1.581 double elapsed_time_ms = (end_time_sec - start_time_sec) * 1000.0;
1.582
1.583 -#ifdef TRACE_CALC_YOUNG_CONFIG
1.584 +#ifdef TRACE_CALC_YOUNG_LENGTH
1.585 // leave this in for debugging, just in case
1.586 - gclog_or_tty->print_cr("target = %1.1lf ms, young = " SIZE_FORMAT
1.587 - ", SO = " SIZE_FORMAT ", "
1.588 - "elapsed %1.2lf ms, calcs: " SIZE_FORMAT " (%s%s) "
1.589 - SIZE_FORMAT SIZE_FORMAT,
1.590 + gclog_or_tty->print_cr("target = %1.1lf ms, young = " SIZE_FORMAT ", "
1.591 + "elapsed %1.2lf ms, (%s%s) " SIZE_FORMAT SIZE_FORMAT,
1.592 target_pause_time_ms,
1.593 - _young_list_target_length - _young_list_so_prefix_length,
1.594 - _young_list_so_prefix_length,
1.595 + _young_list_target_length
1.596 elapsed_time_ms,
1.597 - calculations,
1.598 full_young_gcs() ? "full" : "partial",
1.599 during_initial_mark_pause() ? " i-m" : "",
1.600 _in_marking_window,
1.601 _in_marking_window_im);
1.602 -#endif // TRACE_CALC_YOUNG_CONFIG
1.603 +#endif // TRACE_CALC_YOUNG_LENGTH
1.604
1.605 if (_young_list_target_length < _young_list_min_length) {
1.606 - // bummer; this means that, if we do a pause when the optimal
1.607 - // config dictates, we'll violate the pause spacing target (the
1.608 + // bummer; this means that, if we do a pause when the maximal
1.609 + // length dictates, we'll violate the pause spacing target (the
1.610 // min length was calculate based on the application's current
1.611 // alloc rate);
1.612
1.613 // so, we have to bite the bullet, and allocate the minimum
1.614 // number. We'll violate our target, but we just can't meet it.
1.615
1.616 - size_t so_length = 0;
1.617 - // a note further up explains why we do not want an S-O length
1.618 - // during marking
1.619 - if (!_in_marking_window && !_last_full_young_gc)
1.620 - // but we can still try to see whether we can find an optimal
1.621 - // S-O length
1.622 - so_length = calculate_optimal_so_length(_young_list_min_length);
1.623 -
1.624 -#ifdef TRACE_CALC_YOUNG_CONFIG
1.625 +#ifdef TRACE_CALC_YOUNG_LENGTH
1.626 // leave this in for debugging, just in case
1.627 gclog_or_tty->print_cr("adjusted target length from "
1.628 - SIZE_FORMAT " to " SIZE_FORMAT
1.629 - ", SO " SIZE_FORMAT,
1.630 - _young_list_target_length, _young_list_min_length,
1.631 - so_length);
1.632 -#endif // TRACE_CALC_YOUNG_CONFIG
1.633 -
1.634 - _young_list_target_length =
1.635 - MAX2(_young_list_min_length, (size_t)1);
1.636 - _young_list_so_prefix_length = so_length;
1.637 + SIZE_FORMAT " to " SIZE_FORMAT,
1.638 + _young_list_target_length, _young_list_min_length);
1.639 +#endif // TRACE_CALC_YOUNG_LENGTH
1.640 +
1.641 + _young_list_target_length = _young_list_min_length;
1.642 }
1.643 } else {
1.644 // we are in a partially-young mode or we've run out of regions (due
1.645 // to evacuation failure)
1.646
1.647 -#ifdef TRACE_CALC_YOUNG_CONFIG
1.648 +#ifdef TRACE_CALC_YOUNG_LENGTH
1.649 // leave this in for debugging, just in case
1.650 gclog_or_tty->print_cr("(partial) setting target to " SIZE_FORMAT
1.651 - ", SO " SIZE_FORMAT,
1.652 - _young_list_min_length, 0);
1.653 -#endif // TRACE_CALC_YOUNG_CONFIG
1.654 -
1.655 - // we'll do the pause as soon as possible and with no S-O prefix
1.656 - // (see above for the reasons behind the latter)
1.657 + _young_list_min_length);
1.658 +#endif // TRACE_CALC_YOUNG_LENGTH
1.659 + // we'll do the pause as soon as possible by choosing the minimum
1.660 _young_list_target_length =
1.661 MAX2(_young_list_min_length, (size_t) 1);
1.662 - _young_list_so_prefix_length = 0;
1.663 }
1.664
1.665 _rs_lengths_prediction = rs_lengths;
1.666 }
1.667
1.668 -// This is used by: calculate_optimal_so_length(length). It returns
1.669 -// the GC eff and predicted pause time for a particular config
1.670 -void
1.671 -G1CollectorPolicy::predict_gc_eff(size_t young_length,
1.672 - size_t so_length,
1.673 - double base_time_ms,
1.674 - double* ret_gc_eff,
1.675 - double* ret_pause_time_ms) {
1.676 - double so_time_ms = predict_scan_only_time_ms(so_length);
1.677 - double accum_surv_rate_adj = 0.0;
1.678 - if (so_length > 0)
1.679 - accum_surv_rate_adj = accum_yg_surv_rate_pred((int)(so_length - 1));
1.680 - double accum_surv_rate =
1.681 - accum_yg_surv_rate_pred((int)(young_length - 1)) - accum_surv_rate_adj;
1.682 - size_t bytes_to_copy =
1.683 - (size_t) (accum_surv_rate * (double) HeapRegion::GrainBytes);
1.684 - double copy_time_ms = predict_object_copy_time_ms(bytes_to_copy);
1.685 - double young_other_time_ms =
1.686 - predict_young_other_time_ms(young_length - so_length);
1.687 - double pause_time_ms =
1.688 - base_time_ms + so_time_ms + copy_time_ms + young_other_time_ms;
1.689 - size_t reclaimed_bytes =
1.690 - (young_length - so_length) * HeapRegion::GrainBytes - bytes_to_copy;
1.691 - double gc_eff = (double) reclaimed_bytes / pause_time_ms;
1.692 -
1.693 - *ret_gc_eff = gc_eff;
1.694 - *ret_pause_time_ms = pause_time_ms;
1.695 -}
1.696 -
1.697 -// This is used by: calculate_young_list_target_config(rs_length). It
1.698 -// returns the GC eff of a particular config. It returns false if that
1.699 -// config violates any of the end conditions of the search in the
1.700 -// calling method, or true upon success. The end conditions were put
1.701 -// here since it's called twice and it was best not to replicate them
1.702 -// in the caller. Also, passing the parameteres avoids having to
1.703 -// recalculate them in the innermost loop.
1.704 +// This is used by: calculate_young_list_target_length(rs_length). It
1.705 +// returns true iff:
1.706 +// the predicted pause time for the given young list will not overflow
1.707 +// the target pause time
1.708 +// and:
1.709 +// the predicted amount of surviving data will not overflow the
1.710 +// the amount of free space available for survivor regions.
1.711 +//
1.712 bool
1.713 -G1CollectorPolicy::predict_gc_eff(size_t young_length,
1.714 - size_t so_length,
1.715 - double base_time_with_so_ms,
1.716 - size_t init_free_regions,
1.717 - double target_pause_time_ms,
1.718 - double* ret_gc_eff) {
1.719 - *ret_gc_eff = 0.0;
1.720 +G1CollectorPolicy::predict_will_fit(size_t young_length,
1.721 + double base_time_ms,
1.722 + size_t init_free_regions,
1.723 + double target_pause_time_ms) {
1.724
1.725 if (young_length >= init_free_regions)
1.726 // end condition 1: not enough space for the young regions
1.727 return false;
1.728
1.729 double accum_surv_rate_adj = 0.0;
1.730 - if (so_length > 0)
1.731 - accum_surv_rate_adj = accum_yg_surv_rate_pred((int)(so_length - 1));
1.732 double accum_surv_rate =
1.733 accum_yg_surv_rate_pred((int)(young_length - 1)) - accum_surv_rate_adj;
1.734 +
1.735 size_t bytes_to_copy =
1.736 (size_t) (accum_surv_rate * (double) HeapRegion::GrainBytes);
1.737 +
1.738 double copy_time_ms = predict_object_copy_time_ms(bytes_to_copy);
1.739 +
1.740 double young_other_time_ms =
1.741 - predict_young_other_time_ms(young_length - so_length);
1.742 + predict_young_other_time_ms(young_length);
1.743 +
1.744 double pause_time_ms =
1.745 - base_time_with_so_ms + copy_time_ms + young_other_time_ms;
1.746 + base_time_ms + copy_time_ms + young_other_time_ms;
1.747
1.748 if (pause_time_ms > target_pause_time_ms)
1.749 // end condition 2: over the target pause time
1.750 return false;
1.751
1.752 - size_t reclaimed_bytes =
1.753 - (young_length - so_length) * HeapRegion::GrainBytes - bytes_to_copy;
1.754 size_t free_bytes =
1.755 (init_free_regions - young_length) * HeapRegion::GrainBytes;
1.756
1.757 @@ -934,9 +604,6 @@
1.758 return false;
1.759
1.760 // success!
1.761 - double gc_eff = (double) reclaimed_bytes / pause_time_ms;
1.762 - *ret_gc_eff = gc_eff;
1.763 -
1.764 return true;
1.765 }
1.766
1.767 @@ -953,11 +620,11 @@
1.768 void G1CollectorPolicy::check_prediction_validity() {
1.769 guarantee( adaptive_young_list_length(), "should not call this otherwise" );
1.770
1.771 - size_t rs_lengths = _g1->young_list_sampled_rs_lengths();
1.772 + size_t rs_lengths = _g1->young_list()->sampled_rs_lengths();
1.773 if (rs_lengths > _rs_lengths_prediction) {
1.774 // add 10% to avoid having to recalculate often
1.775 size_t rs_lengths_prediction = rs_lengths * 1100 / 1000;
1.776 - calculate_young_list_target_config(rs_lengths_prediction);
1.777 + calculate_young_list_target_length(rs_lengths_prediction);
1.778 }
1.779 }
1.780
1.781 @@ -979,7 +646,7 @@
1.782
1.783 #ifndef PRODUCT
1.784 bool G1CollectorPolicy::verify_young_ages() {
1.785 - HeapRegion* head = _g1->young_list_first_region();
1.786 + HeapRegion* head = _g1->young_list()->first_region();
1.787 return
1.788 verify_young_ages(head, _short_lived_surv_rate_group);
1.789 // also call verify_young_ages on any additional surv rate groups
1.790 @@ -1056,7 +723,6 @@
1.791 _in_marking_window = false;
1.792 _in_marking_window_im = false;
1.793
1.794 - _short_lived_surv_rate_group->record_scan_only_prefix(0);
1.795 _short_lived_surv_rate_group->start_adding_regions();
1.796 // also call this on any additional surv rate groups
1.797
1.798 @@ -1066,11 +732,10 @@
1.799 _prev_region_num_tenured = _region_num_tenured;
1.800
1.801 _free_regions_at_end_of_collection = _g1->free_regions();
1.802 - _scan_only_regions_at_end_of_collection = 0;
1.803 // Reset survivors SurvRateGroup.
1.804 _survivor_surv_rate_group->reset();
1.805 calculate_young_list_min_length();
1.806 - calculate_young_list_target_config();
1.807 + calculate_young_list_target_length();
1.808 }
1.809
1.810 void G1CollectorPolicy::record_before_bytes(size_t bytes) {
1.811 @@ -1119,8 +784,6 @@
1.812 for (int i = 0; i < _parallel_gc_threads; ++i) {
1.813 _par_last_ext_root_scan_times_ms[i] = -666.0;
1.814 _par_last_mark_stack_scan_times_ms[i] = -666.0;
1.815 - _par_last_scan_only_times_ms[i] = -666.0;
1.816 - _par_last_scan_only_regions_scanned[i] = -666.0;
1.817 _par_last_update_rs_start_times_ms[i] = -666.0;
1.818 _par_last_update_rs_times_ms[i] = -666.0;
1.819 _par_last_update_rs_processed_buffers[i] = -666.0;
1.820 @@ -1143,47 +806,13 @@
1.821 if (in_young_gc_mode())
1.822 _last_young_gc_full = false;
1.823
1.824 -
1.825 // do that for any other surv rate groups
1.826 _short_lived_surv_rate_group->stop_adding_regions();
1.827 - size_t short_lived_so_length = _young_list_so_prefix_length;
1.828 - _short_lived_surv_rate_group->record_scan_only_prefix(short_lived_so_length);
1.829 - tag_scan_only(short_lived_so_length);
1.830 _survivors_age_table.clear();
1.831
1.832 assert( verify_young_ages(), "region age verification" );
1.833 }
1.834
1.835 -void G1CollectorPolicy::tag_scan_only(size_t short_lived_scan_only_length) {
1.836 - // done in a way that it can be extended for other surv rate groups too...
1.837 -
1.838 - HeapRegion* head = _g1->young_list_first_region();
1.839 - bool finished_short_lived = (short_lived_scan_only_length == 0);
1.840 -
1.841 - if (finished_short_lived)
1.842 - return;
1.843 -
1.844 - for (HeapRegion* curr = head;
1.845 - curr != NULL;
1.846 - curr = curr->get_next_young_region()) {
1.847 - SurvRateGroup* surv_rate_group = curr->surv_rate_group();
1.848 - int age = curr->age_in_surv_rate_group();
1.849 -
1.850 - if (surv_rate_group == _short_lived_surv_rate_group) {
1.851 - if ((size_t)age < short_lived_scan_only_length)
1.852 - curr->set_scan_only();
1.853 - else
1.854 - finished_short_lived = true;
1.855 - }
1.856 -
1.857 -
1.858 - if (finished_short_lived)
1.859 - return;
1.860 - }
1.861 -
1.862 - guarantee( false, "we should never reach here" );
1.863 -}
1.864 -
1.865 void G1CollectorPolicy::record_mark_closure_time(double mark_closure_time_ms) {
1.866 _mark_closure_time_ms = mark_closure_time_ms;
1.867 }
1.868 @@ -1277,7 +906,7 @@
1.869 _last_full_young_gc = true;
1.870 _in_marking_window = false;
1.871 if (adaptive_young_list_length())
1.872 - calculate_young_list_target_config();
1.873 + calculate_young_list_target_length();
1.874 }
1.875 }
1.876
1.877 @@ -1512,6 +1141,7 @@
1.878 size_t freed_bytes =
1.879 _cur_collection_pause_used_at_start_bytes - cur_used_bytes;
1.880 size_t surviving_bytes = _collection_set_bytes_used_before - freed_bytes;
1.881 +
1.882 double survival_fraction =
1.883 (double)surviving_bytes/
1.884 (double)_collection_set_bytes_used_before;
1.885 @@ -1599,9 +1229,6 @@
1.886
1.887 double ext_root_scan_time = avg_value(_par_last_ext_root_scan_times_ms);
1.888 double mark_stack_scan_time = avg_value(_par_last_mark_stack_scan_times_ms);
1.889 - double scan_only_time = avg_value(_par_last_scan_only_times_ms);
1.890 - double scan_only_regions_scanned =
1.891 - sum_of_values(_par_last_scan_only_regions_scanned);
1.892 double update_rs_time = avg_value(_par_last_update_rs_times_ms);
1.893 double update_rs_processed_buffers =
1.894 sum_of_values(_par_last_update_rs_processed_buffers);
1.895 @@ -1611,7 +1238,7 @@
1.896
1.897 double parallel_other_time = _cur_collection_par_time_ms -
1.898 (update_rs_time + ext_root_scan_time + mark_stack_scan_time +
1.899 - scan_only_time + scan_rs_time + obj_copy_time + termination_time);
1.900 + scan_rs_time + obj_copy_time + termination_time);
1.901 if (update_stats) {
1.902 MainBodySummary* body_summary = summary->main_body_summary();
1.903 guarantee(body_summary != NULL, "should not be null!");
1.904 @@ -1622,7 +1249,6 @@
1.905 body_summary->record_satb_drain_time_ms(0.0);
1.906 body_summary->record_ext_root_scan_time_ms(ext_root_scan_time);
1.907 body_summary->record_mark_stack_scan_time_ms(mark_stack_scan_time);
1.908 - body_summary->record_scan_only_time_ms(scan_only_time);
1.909 body_summary->record_update_rs_time_ms(update_rs_time);
1.910 body_summary->record_scan_rs_time_ms(scan_rs_time);
1.911 body_summary->record_obj_copy_time_ms(obj_copy_time);
1.912 @@ -1676,7 +1302,7 @@
1.913 else
1.914 other_time_ms -=
1.915 update_rs_time +
1.916 - ext_root_scan_time + mark_stack_scan_time + scan_only_time +
1.917 + ext_root_scan_time + mark_stack_scan_time +
1.918 scan_rs_time + obj_copy_time;
1.919 }
1.920
1.921 @@ -1701,9 +1327,6 @@
1.922 _par_last_update_rs_processed_buffers, true);
1.923 print_par_stats(2, "Ext Root Scanning", _par_last_ext_root_scan_times_ms);
1.924 print_par_stats(2, "Mark Stack Scanning", _par_last_mark_stack_scan_times_ms);
1.925 - print_par_stats(2, "Scan-Only Scanning", _par_last_scan_only_times_ms);
1.926 - print_par_buffers(3, "Scan-Only Regions",
1.927 - _par_last_scan_only_regions_scanned, true);
1.928 print_par_stats(2, "Scan RS", _par_last_scan_rs_times_ms);
1.929 print_par_stats(2, "Object Copy", _par_last_obj_copy_times_ms);
1.930 print_par_stats(2, "Termination", _par_last_termination_times_ms);
1.931 @@ -1715,7 +1338,6 @@
1.932 (int)update_rs_processed_buffers);
1.933 print_stats(1, "Ext Root Scanning", ext_root_scan_time);
1.934 print_stats(1, "Mark Stack Scanning", mark_stack_scan_time);
1.935 - print_stats(1, "Scan-Only Scanning", scan_only_time);
1.936 print_stats(1, "Scan RS", scan_rs_time);
1.937 print_stats(1, "Object Copying", obj_copy_time);
1.938 }
1.939 @@ -1730,6 +1352,8 @@
1.940 }
1.941 #endif
1.942 print_stats(1, "Other", other_time_ms);
1.943 + print_stats(2, "Choose CSet", _recorded_young_cset_choice_time_ms);
1.944 +
1.945 for (int i = 0; i < _aux_num; ++i) {
1.946 if (_cur_aux_times_set[i]) {
1.947 char buffer[96];
1.948 @@ -1815,16 +1439,6 @@
1.949 _cost_per_card_ms_seq->add(cost_per_card_ms);
1.950 }
1.951
1.952 - double cost_per_scan_only_region_ms = 0.0;
1.953 - if (scan_only_regions_scanned > 0.0) {
1.954 - cost_per_scan_only_region_ms =
1.955 - scan_only_time / scan_only_regions_scanned;
1.956 - if (_in_marking_window_im)
1.957 - _cost_per_scan_only_region_ms_during_cm_seq->add(cost_per_scan_only_region_ms);
1.958 - else
1.959 - _cost_per_scan_only_region_ms_seq->add(cost_per_scan_only_region_ms);
1.960 - }
1.961 -
1.962 size_t cards_scanned = _g1->cards_scanned();
1.963
1.964 double cost_per_entry_ms = 0.0;
1.965 @@ -1860,7 +1474,7 @@
1.966 }
1.967
1.968 double all_other_time_ms = pause_time_ms -
1.969 - (update_rs_time + scan_only_time + scan_rs_time + obj_copy_time +
1.970 + (update_rs_time + scan_rs_time + obj_copy_time +
1.971 _mark_closure_time_ms + termination_time);
1.972
1.973 double young_other_time_ms = 0.0;
1.974 @@ -1907,11 +1521,10 @@
1.975 if (PREDICTIONS_VERBOSE) {
1.976 gclog_or_tty->print_cr("");
1.977 gclog_or_tty->print_cr("PREDICTIONS %1.4lf %d "
1.978 - "REGIONS %d %d %d %d "
1.979 + "REGIONS %d %d %d "
1.980 "PENDING_CARDS %d %d "
1.981 "CARDS_SCANNED %d %d "
1.982 "RS_LENGTHS %d %d "
1.983 - "SCAN_ONLY_SCAN %1.6lf %1.6lf "
1.984 "RS_UPDATE %1.6lf %1.6lf RS_SCAN %1.6lf %1.6lf "
1.985 "SURVIVAL_RATIO %1.6lf %1.6lf "
1.986 "OBJECT_COPY %1.6lf %1.6lf OTHER_CONSTANT %1.6lf %1.6lf "
1.987 @@ -1924,12 +1537,10 @@
1.988 (last_pause_included_initial_mark) ? 1 : 0,
1.989 _recorded_region_num,
1.990 _recorded_young_regions,
1.991 - _recorded_scan_only_regions,
1.992 _recorded_non_young_regions,
1.993 _predicted_pending_cards, _pending_cards,
1.994 _predicted_cards_scanned, cards_scanned,
1.995 _predicted_rs_lengths, _max_rs_lengths,
1.996 - _predicted_scan_only_scan_time_ms, scan_only_time,
1.997 _predicted_rs_update_time_ms, update_rs_time,
1.998 _predicted_rs_scan_time_ms, scan_rs_time,
1.999 _predicted_survival_ratio, survival_ratio,
1.1000 @@ -1954,14 +1565,12 @@
1.1001 _in_marking_window = new_in_marking_window;
1.1002 _in_marking_window_im = new_in_marking_window_im;
1.1003 _free_regions_at_end_of_collection = _g1->free_regions();
1.1004 - _scan_only_regions_at_end_of_collection = _g1->young_list_length();
1.1005 calculate_young_list_min_length();
1.1006 - calculate_young_list_target_config();
1.1007 + calculate_young_list_target_length();
1.1008
1.1009 // Note that _mmu_tracker->max_gc_time() returns the time in seconds.
1.1010 double update_rs_time_goal_ms = _mmu_tracker->max_gc_time() * MILLIUNITS * G1RSetUpdatingPauseTimePercent / 100.0;
1.1011 adjust_concurrent_refinement(update_rs_time, update_rs_processed_buffers, update_rs_time_goal_ms);
1.1012 -
1.1013 // </NEW PREDICTION>
1.1014
1.1015 _target_pause_time_ms = -1.0;
1.1016 @@ -2016,13 +1625,13 @@
1.1017 guarantee( adjustment == 0 || adjustment == 1, "invariant" );
1.1018
1.1019 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1.1020 - size_t young_num = g1h->young_list_length();
1.1021 + size_t young_num = g1h->young_list()->length();
1.1022 if (young_num == 0)
1.1023 return 0.0;
1.1024
1.1025 young_num += adjustment;
1.1026 size_t pending_cards = predict_pending_cards();
1.1027 - size_t rs_lengths = g1h->young_list_sampled_rs_lengths() +
1.1028 + size_t rs_lengths = g1h->young_list()->sampled_rs_lengths() +
1.1029 predict_rs_length_diff();
1.1030 size_t card_num;
1.1031 if (full_young_gcs())
1.1032 @@ -2106,31 +1715,22 @@
1.1033 void
1.1034 G1CollectorPolicy::start_recording_regions() {
1.1035 _recorded_rs_lengths = 0;
1.1036 - _recorded_scan_only_regions = 0;
1.1037 _recorded_young_regions = 0;
1.1038 _recorded_non_young_regions = 0;
1.1039
1.1040 #if PREDICTIONS_VERBOSE
1.1041 - _predicted_rs_lengths = 0;
1.1042 - _predicted_cards_scanned = 0;
1.1043 -
1.1044 _recorded_marked_bytes = 0;
1.1045 _recorded_young_bytes = 0;
1.1046 _predicted_bytes_to_copy = 0;
1.1047 + _predicted_rs_lengths = 0;
1.1048 + _predicted_cards_scanned = 0;
1.1049 #endif // PREDICTIONS_VERBOSE
1.1050 }
1.1051
1.1052 void
1.1053 -G1CollectorPolicy::record_cset_region(HeapRegion* hr, bool young) {
1.1054 - if (young) {
1.1055 - ++_recorded_young_regions;
1.1056 - } else {
1.1057 - ++_recorded_non_young_regions;
1.1058 - }
1.1059 +G1CollectorPolicy::record_cset_region_info(HeapRegion* hr, bool young) {
1.1060 #if PREDICTIONS_VERBOSE
1.1061 - if (young) {
1.1062 - _recorded_young_bytes += hr->used();
1.1063 - } else {
1.1064 + if (!young) {
1.1065 _recorded_marked_bytes += hr->max_live_bytes();
1.1066 }
1.1067 _predicted_bytes_to_copy += predict_bytes_to_copy(hr);
1.1068 @@ -2141,12 +1741,37 @@
1.1069 }
1.1070
1.1071 void
1.1072 -G1CollectorPolicy::record_scan_only_regions(size_t scan_only_length) {
1.1073 - _recorded_scan_only_regions = scan_only_length;
1.1074 +G1CollectorPolicy::record_non_young_cset_region(HeapRegion* hr) {
1.1075 + assert(!hr->is_young(), "should not call this");
1.1076 + ++_recorded_non_young_regions;
1.1077 + record_cset_region_info(hr, false);
1.1078 +}
1.1079 +
1.1080 +void
1.1081 +G1CollectorPolicy::set_recorded_young_regions(size_t n_regions) {
1.1082 + _recorded_young_regions = n_regions;
1.1083 +}
1.1084 +
1.1085 +void G1CollectorPolicy::set_recorded_young_bytes(size_t bytes) {
1.1086 +#if PREDICTIONS_VERBOSE
1.1087 + _recorded_young_bytes = bytes;
1.1088 +#endif // PREDICTIONS_VERBOSE
1.1089 +}
1.1090 +
1.1091 +void G1CollectorPolicy::set_recorded_rs_lengths(size_t rs_lengths) {
1.1092 + _recorded_rs_lengths = rs_lengths;
1.1093 +}
1.1094 +
1.1095 +void G1CollectorPolicy::set_predicted_bytes_to_copy(size_t bytes) {
1.1096 + _predicted_bytes_to_copy = bytes;
1.1097 }
1.1098
1.1099 void
1.1100 G1CollectorPolicy::end_recording_regions() {
1.1101 + // The _predicted_pause_time_ms field is referenced in code
1.1102 + // not under PREDICTIONS_VERBOSE. Let's initialize it.
1.1103 + _predicted_pause_time_ms = -1.0;
1.1104 +
1.1105 #if PREDICTIONS_VERBOSE
1.1106 _predicted_pending_cards = predict_pending_cards();
1.1107 _predicted_rs_lengths = _recorded_rs_lengths + predict_rs_length_diff();
1.1108 @@ -2157,8 +1782,6 @@
1.1109 predict_non_young_card_num(_predicted_rs_lengths);
1.1110 _recorded_region_num = _recorded_young_regions + _recorded_non_young_regions;
1.1111
1.1112 - _predicted_scan_only_scan_time_ms =
1.1113 - predict_scan_only_time_ms(_recorded_scan_only_regions);
1.1114 _predicted_rs_update_time_ms =
1.1115 predict_rs_update_time_ms(_g1->pending_card_num());
1.1116 _predicted_rs_scan_time_ms =
1.1117 @@ -2173,7 +1796,6 @@
1.1118 predict_non_young_other_time_ms(_recorded_non_young_regions);
1.1119
1.1120 _predicted_pause_time_ms =
1.1121 - _predicted_scan_only_scan_time_ms +
1.1122 _predicted_rs_update_time_ms +
1.1123 _predicted_rs_scan_time_ms +
1.1124 _predicted_object_copy_time_ms +
1.1125 @@ -2463,8 +2085,6 @@
1.1126 body_summary->get_ext_root_scan_seq());
1.1127 print_summary(2, "Mark Stack Scanning",
1.1128 body_summary->get_mark_stack_scan_seq());
1.1129 - print_summary(2, "Scan-Only Scanning",
1.1130 - body_summary->get_scan_only_seq());
1.1131 print_summary(2, "Scan RS", body_summary->get_scan_rs_seq());
1.1132 print_summary(2, "Object Copy", body_summary->get_obj_copy_seq());
1.1133 print_summary(2, "Termination", body_summary->get_termination_seq());
1.1134 @@ -2474,7 +2094,6 @@
1.1135 body_summary->get_update_rs_seq(),
1.1136 body_summary->get_ext_root_scan_seq(),
1.1137 body_summary->get_mark_stack_scan_seq(),
1.1138 - body_summary->get_scan_only_seq(),
1.1139 body_summary->get_scan_rs_seq(),
1.1140 body_summary->get_obj_copy_seq(),
1.1141 body_summary->get_termination_seq()
1.1142 @@ -2492,8 +2111,6 @@
1.1143 body_summary->get_ext_root_scan_seq());
1.1144 print_summary(1, "Mark Stack Scanning",
1.1145 body_summary->get_mark_stack_scan_seq());
1.1146 - print_summary(1, "Scan-Only Scanning",
1.1147 - body_summary->get_scan_only_seq());
1.1148 print_summary(1, "Scan RS", body_summary->get_scan_rs_seq());
1.1149 print_summary(1, "Object Copy", body_summary->get_obj_copy_seq());
1.1150 }
1.1151 @@ -2519,7 +2136,6 @@
1.1152 body_summary->get_update_rs_seq(),
1.1153 body_summary->get_ext_root_scan_seq(),
1.1154 body_summary->get_mark_stack_scan_seq(),
1.1155 - body_summary->get_scan_only_seq(),
1.1156 body_summary->get_scan_rs_seq(),
1.1157 body_summary->get_obj_copy_seq()
1.1158 };
1.1159 @@ -2613,7 +2229,7 @@
1.1160 G1CollectorPolicy::should_add_next_region_to_young_list() {
1.1161 assert(in_young_gc_mode(), "should be in young GC mode");
1.1162 bool ret;
1.1163 - size_t young_list_length = _g1->young_list_length();
1.1164 + size_t young_list_length = _g1->young_list()->length();
1.1165 size_t young_list_max_length = _young_list_target_length;
1.1166 if (G1FixedEdenSize) {
1.1167 young_list_max_length -= _max_survivor_regions;
1.1168 @@ -2676,7 +2292,7 @@
1.1169 assert(_g1->regions_accounted_for(), "Region leakage!");
1.1170 double max_pause_time_ms = _mmu_tracker->max_gc_time() * 1000.0;
1.1171
1.1172 - size_t young_list_length = _g1->young_list_length();
1.1173 + size_t young_list_length = _g1->young_list()->length();
1.1174 size_t young_list_max_length = _young_list_target_length;
1.1175 if (G1FixedEdenSize) {
1.1176 young_list_max_length -= _max_survivor_regions;
1.1177 @@ -2685,7 +2301,7 @@
1.1178
1.1179 if (in_young_gc_mode()) {
1.1180 if (reached_target_length) {
1.1181 - assert( young_list_length > 0 && _g1->young_list_length() > 0,
1.1182 + assert( young_list_length > 0 && _g1->young_list()->length() > 0,
1.1183 "invariant" );
1.1184 _target_pause_time_ms = max_pause_time_ms;
1.1185 return true;
1.1186 @@ -2946,10 +2562,12 @@
1.1187 }
1.1188 }
1.1189
1.1190 -// Add the heap region to the collection set and return the conservative
1.1191 -// estimate of the number of live bytes.
1.1192 +// Add the heap region at the head of the non-incremental collection set
1.1193 void G1CollectorPolicy::
1.1194 add_to_collection_set(HeapRegion* hr) {
1.1195 + assert(_inc_cset_build_state == Active, "Precondition");
1.1196 + assert(!hr->is_young(), "non-incremental add of young region");
1.1197 +
1.1198 if (G1PrintHeapRegions) {
1.1199 gclog_or_tty->print_cr("added region to cset "
1.1200 "%d:["PTR_FORMAT", "PTR_FORMAT"], "
1.1201 @@ -2961,8 +2579,7 @@
1.1202 if (_g1->mark_in_progress())
1.1203 _g1->concurrent_mark()->registerCSetRegion(hr);
1.1204
1.1205 - assert(!hr->in_collection_set(),
1.1206 - "should not already be in the CSet");
1.1207 + assert(!hr->in_collection_set(), "should not already be in the CSet");
1.1208 hr->set_in_collection_set(true);
1.1209 hr->set_next_in_collection_set(_collection_set);
1.1210 _collection_set = hr;
1.1211 @@ -2971,10 +2588,230 @@
1.1212 _g1->register_region_with_in_cset_fast_test(hr);
1.1213 }
1.1214
1.1215 -void
1.1216 -G1CollectorPolicy_BestRegionsFirst::
1.1217 -choose_collection_set() {
1.1218 - double non_young_start_time_sec;
1.1219 +// Initialize the per-collection-set information
1.1220 +void G1CollectorPolicy::start_incremental_cset_building() {
1.1221 + assert(_inc_cset_build_state == Inactive, "Precondition");
1.1222 +
1.1223 + _inc_cset_head = NULL;
1.1224 + _inc_cset_tail = NULL;
1.1225 + _inc_cset_size = 0;
1.1226 + _inc_cset_bytes_used_before = 0;
1.1227 +
1.1228 + if (in_young_gc_mode()) {
1.1229 + _inc_cset_young_index = 0;
1.1230 + }
1.1231 +
1.1232 + _inc_cset_max_finger = 0;
1.1233 + _inc_cset_recorded_young_bytes = 0;
1.1234 + _inc_cset_recorded_rs_lengths = 0;
1.1235 + _inc_cset_predicted_elapsed_time_ms = 0;
1.1236 + _inc_cset_predicted_bytes_to_copy = 0;
1.1237 + _inc_cset_build_state = Active;
1.1238 +}
1.1239 +
1.1240 +void G1CollectorPolicy::add_to_incremental_cset_info(HeapRegion* hr, size_t rs_length) {
1.1241 + // This routine is used when:
1.1242 + // * adding survivor regions to the incremental cset at the end of an
1.1243 + // evacuation pause,
1.1244 + // * adding the current allocation region to the incremental cset
1.1245 + // when it is retired, and
1.1246 + // * updating existing policy information for a region in the
1.1247 + // incremental cset via young list RSet sampling.
1.1248 + // Therefore this routine may be called at a safepoint by the
1.1249 + // VM thread, or in-between safepoints by mutator threads (when
1.1250 + // retiring the current allocation region) or a concurrent
1.1251 + // refine thread (RSet sampling).
1.1252 +
1.1253 + double region_elapsed_time_ms = predict_region_elapsed_time_ms(hr, true);
1.1254 + size_t used_bytes = hr->used();
1.1255 +
1.1256 + _inc_cset_recorded_rs_lengths += rs_length;
1.1257 + _inc_cset_predicted_elapsed_time_ms += region_elapsed_time_ms;
1.1258 +
1.1259 + _inc_cset_bytes_used_before += used_bytes;
1.1260 +
1.1261 + // Cache the values we have added to the aggregated informtion
1.1262 + // in the heap region in case we have to remove this region from
1.1263 + // the incremental collection set, or it is updated by the
1.1264 + // rset sampling code
1.1265 + hr->set_recorded_rs_length(rs_length);
1.1266 + hr->set_predicted_elapsed_time_ms(region_elapsed_time_ms);
1.1267 +
1.1268 +#if PREDICTIONS_VERBOSE
1.1269 + size_t bytes_to_copy = predict_bytes_to_copy(hr);
1.1270 + _inc_cset_predicted_bytes_to_copy += bytes_to_copy;
1.1271 +
1.1272 + // Record the number of bytes used in this region
1.1273 + _inc_cset_recorded_young_bytes += used_bytes;
1.1274 +
1.1275 + // Cache the values we have added to the aggregated informtion
1.1276 + // in the heap region in case we have to remove this region from
1.1277 + // the incremental collection set, or it is updated by the
1.1278 + // rset sampling code
1.1279 + hr->set_predicted_bytes_to_copy(bytes_to_copy);
1.1280 +#endif // PREDICTIONS_VERBOSE
1.1281 +}
1.1282 +
1.1283 +void G1CollectorPolicy::remove_from_incremental_cset_info(HeapRegion* hr) {
1.1284 + // This routine is currently only called as part of the updating of
1.1285 + // existing policy information for regions in the incremental cset that
1.1286 + // is performed by the concurrent refine thread(s) as part of young list
1.1287 + // RSet sampling. Therefore we should not be at a safepoint.
1.1288 +
1.1289 + assert(!SafepointSynchronize::is_at_safepoint(), "should not be at safepoint");
1.1290 + assert(hr->is_young(), "it should be");
1.1291 +
1.1292 + size_t used_bytes = hr->used();
1.1293 + size_t old_rs_length = hr->recorded_rs_length();
1.1294 + double old_elapsed_time_ms = hr->predicted_elapsed_time_ms();
1.1295 +
1.1296 + // Subtract the old recorded/predicted policy information for
1.1297 + // the given heap region from the collection set info.
1.1298 + _inc_cset_recorded_rs_lengths -= old_rs_length;
1.1299 + _inc_cset_predicted_elapsed_time_ms -= old_elapsed_time_ms;
1.1300 +
1.1301 + _inc_cset_bytes_used_before -= used_bytes;
1.1302 +
1.1303 + // Clear the values cached in the heap region
1.1304 + hr->set_recorded_rs_length(0);
1.1305 + hr->set_predicted_elapsed_time_ms(0);
1.1306 +
1.1307 +#if PREDICTIONS_VERBOSE
1.1308 + size_t old_predicted_bytes_to_copy = hr->predicted_bytes_to_copy();
1.1309 + _inc_cset_predicted_bytes_to_copy -= old_predicted_bytes_to_copy;
1.1310 +
1.1311 + // Subtract the number of bytes used in this region
1.1312 + _inc_cset_recorded_young_bytes -= used_bytes;
1.1313 +
1.1314 + // Clear the values cached in the heap region
1.1315 + hr->set_predicted_bytes_to_copy(0);
1.1316 +#endif // PREDICTIONS_VERBOSE
1.1317 +}
1.1318 +
1.1319 +void G1CollectorPolicy::update_incremental_cset_info(HeapRegion* hr, size_t new_rs_length) {
1.1320 + // Update the collection set information that is dependent on the new RS length
1.1321 + assert(hr->is_young(), "Precondition");
1.1322 +
1.1323 + remove_from_incremental_cset_info(hr);
1.1324 + add_to_incremental_cset_info(hr, new_rs_length);
1.1325 +}
1.1326 +
1.1327 +void G1CollectorPolicy::add_region_to_incremental_cset_common(HeapRegion* hr) {
1.1328 + assert( hr->is_young(), "invariant");
1.1329 + assert( hr->young_index_in_cset() == -1, "invariant" );
1.1330 + assert(_inc_cset_build_state == Active, "Precondition");
1.1331 +
1.1332 + // We need to clear and set the cached recorded/cached collection set
1.1333 + // information in the heap region here (before the region gets added
1.1334 + // to the collection set). An individual heap region's cached values
1.1335 + // are calculated, aggregated with the policy collection set info,
1.1336 + // and cached in the heap region here (initially) and (subsequently)
1.1337 + // by the Young List sampling code.
1.1338 +
1.1339 + size_t rs_length = hr->rem_set()->occupied();
1.1340 + add_to_incremental_cset_info(hr, rs_length);
1.1341 +
1.1342 + HeapWord* hr_end = hr->end();
1.1343 + _inc_cset_max_finger = MAX2(_inc_cset_max_finger, hr_end);
1.1344 +
1.1345 + assert(!hr->in_collection_set(), "invariant");
1.1346 + hr->set_in_collection_set(true);
1.1347 + assert( hr->next_in_collection_set() == NULL, "invariant");
1.1348 +
1.1349 + _inc_cset_size++;
1.1350 + _g1->register_region_with_in_cset_fast_test(hr);
1.1351 +
1.1352 + hr->set_young_index_in_cset((int) _inc_cset_young_index);
1.1353 + ++_inc_cset_young_index;
1.1354 +}
1.1355 +
1.1356 +// Add the region at the RHS of the incremental cset
1.1357 +void G1CollectorPolicy::add_region_to_incremental_cset_rhs(HeapRegion* hr) {
1.1358 + // We should only ever be appending survivors at the end of a pause
1.1359 + assert( hr->is_survivor(), "Logic");
1.1360 +
1.1361 + // Do the 'common' stuff
1.1362 + add_region_to_incremental_cset_common(hr);
1.1363 +
1.1364 + // Now add the region at the right hand side
1.1365 + if (_inc_cset_tail == NULL) {
1.1366 + assert(_inc_cset_head == NULL, "invariant");
1.1367 + _inc_cset_head = hr;
1.1368 + } else {
1.1369 + _inc_cset_tail->set_next_in_collection_set(hr);
1.1370 + }
1.1371 + _inc_cset_tail = hr;
1.1372 +
1.1373 + if (G1PrintHeapRegions) {
1.1374 + gclog_or_tty->print_cr(" added region to incremental cset (RHS) "
1.1375 + "%d:["PTR_FORMAT", "PTR_FORMAT"], "
1.1376 + "top "PTR_FORMAT", young %s",
1.1377 + hr->hrs_index(), hr->bottom(), hr->end(),
1.1378 + hr->top(), (hr->is_young()) ? "YES" : "NO");
1.1379 + }
1.1380 +}
1.1381 +
1.1382 +// Add the region to the LHS of the incremental cset
1.1383 +void G1CollectorPolicy::add_region_to_incremental_cset_lhs(HeapRegion* hr) {
1.1384 + // Survivors should be added to the RHS at the end of a pause
1.1385 + assert(!hr->is_survivor(), "Logic");
1.1386 +
1.1387 + // Do the 'common' stuff
1.1388 + add_region_to_incremental_cset_common(hr);
1.1389 +
1.1390 + // Add the region at the left hand side
1.1391 + hr->set_next_in_collection_set(_inc_cset_head);
1.1392 + if (_inc_cset_head == NULL) {
1.1393 + assert(_inc_cset_tail == NULL, "Invariant");
1.1394 + _inc_cset_tail = hr;
1.1395 + }
1.1396 + _inc_cset_head = hr;
1.1397 +
1.1398 + if (G1PrintHeapRegions) {
1.1399 + gclog_or_tty->print_cr(" added region to incremental cset (LHS) "
1.1400 + "%d:["PTR_FORMAT", "PTR_FORMAT"], "
1.1401 + "top "PTR_FORMAT", young %s",
1.1402 + hr->hrs_index(), hr->bottom(), hr->end(),
1.1403 + hr->top(), (hr->is_young()) ? "YES" : "NO");
1.1404 + }
1.1405 +}
1.1406 +
1.1407 +#ifndef PRODUCT
1.1408 +void G1CollectorPolicy::print_collection_set(HeapRegion* list_head, outputStream* st) {
1.1409 + assert(list_head == inc_cset_head() || list_head == collection_set(), "must be");
1.1410 +
1.1411 + st->print_cr("\nCollection_set:");
1.1412 + HeapRegion* csr = list_head;
1.1413 + while (csr != NULL) {
1.1414 + HeapRegion* next = csr->next_in_collection_set();
1.1415 + assert(csr->in_collection_set(), "bad CS");
1.1416 + st->print_cr(" [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, "
1.1417 + "age: %4d, y: %d, surv: %d",
1.1418 + csr->bottom(), csr->end(),
1.1419 + csr->top(),
1.1420 + csr->prev_top_at_mark_start(),
1.1421 + csr->next_top_at_mark_start(),
1.1422 + csr->top_at_conc_mark_count(),
1.1423 + csr->age_in_surv_rate_group_cond(),
1.1424 + csr->is_young(),
1.1425 + csr->is_survivor());
1.1426 + csr = next;
1.1427 + }
1.1428 +}
1.1429 +#endif // !PRODUCT
1.1430 +
1.1431 +bool
1.1432 +G1CollectorPolicy_BestRegionsFirst::choose_collection_set() {
1.1433 + // Set this here - in case we're not doing young collections.
1.1434 + double non_young_start_time_sec = os::elapsedTime();
1.1435 +
1.1436 + // The result that this routine will return. This will be set to
1.1437 + // false if:
1.1438 + // * we're doing a young or partially young collection and we
1.1439 + // have added the youg regions to collection set, or
1.1440 + // * we add old regions to the collection set.
1.1441 + bool abandon_collection = true;
1.1442 +
1.1443 start_recording_regions();
1.1444
1.1445 guarantee(_target_pause_time_ms > -1.0
1.1446 @@ -3027,47 +2864,79 @@
1.1447
1.1448 if (G1PolicyVerbose > 0) {
1.1449 gclog_or_tty->print_cr("Adding %d young regions to the CSet",
1.1450 - _g1->young_list_length());
1.1451 + _g1->young_list()->length());
1.1452 }
1.1453 +
1.1454 _young_cset_length = 0;
1.1455 _last_young_gc_full = full_young_gcs() ? true : false;
1.1456 +
1.1457 if (_last_young_gc_full)
1.1458 ++_full_young_pause_num;
1.1459 else
1.1460 ++_partial_young_pause_num;
1.1461 - hr = _g1->pop_region_from_young_list();
1.1462 +
1.1463 + // The young list is laid with the survivor regions from the previous
1.1464 + // pause are appended to the RHS of the young list, i.e.
1.1465 + // [Newly Young Regions ++ Survivors from last pause].
1.1466 +
1.1467 + hr = _g1->young_list()->first_survivor_region();
1.1468 while (hr != NULL) {
1.1469 -
1.1470 - assert( hr->young_index_in_cset() == -1, "invariant" );
1.1471 - assert( hr->age_in_surv_rate_group() != -1, "invariant" );
1.1472 - hr->set_young_index_in_cset((int) _young_cset_length);
1.1473 -
1.1474 - ++_young_cset_length;
1.1475 - double predicted_time_ms = predict_region_elapsed_time_ms(hr, true);
1.1476 - time_remaining_ms -= predicted_time_ms;
1.1477 - predicted_pause_time_ms += predicted_time_ms;
1.1478 - assert(!hr->in_collection_set(), "invariant");
1.1479 - add_to_collection_set(hr);
1.1480 - record_cset_region(hr, true);
1.1481 - max_live_bytes -= MIN2(hr->max_live_bytes(), max_live_bytes);
1.1482 - if (G1PolicyVerbose > 0) {
1.1483 - gclog_or_tty->print_cr(" Added [" PTR_FORMAT ", " PTR_FORMAT") to CS.",
1.1484 - hr->bottom(), hr->end());
1.1485 - gclog_or_tty->print_cr(" (" SIZE_FORMAT " KB left in heap.)",
1.1486 - max_live_bytes/K);
1.1487 - }
1.1488 - hr = _g1->pop_region_from_young_list();
1.1489 + assert(hr->is_survivor(), "badly formed young list");
1.1490 + hr->set_young();
1.1491 + hr = hr->get_next_young_region();
1.1492 }
1.1493
1.1494 - record_scan_only_regions(_g1->young_list_scan_only_length());
1.1495 + // Clear the fields that point to the survivor list - they are
1.1496 + // all young now.
1.1497 + _g1->young_list()->clear_survivors();
1.1498 +
1.1499 + if (_g1->mark_in_progress())
1.1500 + _g1->concurrent_mark()->register_collection_set_finger(_inc_cset_max_finger);
1.1501 +
1.1502 + _young_cset_length = _inc_cset_young_index;
1.1503 + _collection_set = _inc_cset_head;
1.1504 + _collection_set_size = _inc_cset_size;
1.1505 + _collection_set_bytes_used_before = _inc_cset_bytes_used_before;
1.1506 +
1.1507 + // For young regions in the collection set, we assume the worst
1.1508 + // case of complete survival
1.1509 + max_live_bytes -= _inc_cset_size * HeapRegion::GrainBytes;
1.1510 +
1.1511 + time_remaining_ms -= _inc_cset_predicted_elapsed_time_ms;
1.1512 + predicted_pause_time_ms += _inc_cset_predicted_elapsed_time_ms;
1.1513 +
1.1514 + // The number of recorded young regions is the incremental
1.1515 + // collection set's current size
1.1516 + set_recorded_young_regions(_inc_cset_size);
1.1517 + set_recorded_rs_lengths(_inc_cset_recorded_rs_lengths);
1.1518 + set_recorded_young_bytes(_inc_cset_recorded_young_bytes);
1.1519 +#if PREDICTIONS_VERBOSE
1.1520 + set_predicted_bytes_to_copy(_inc_cset_predicted_bytes_to_copy);
1.1521 +#endif // PREDICTIONS_VERBOSE
1.1522 +
1.1523 + if (G1PolicyVerbose > 0) {
1.1524 + gclog_or_tty->print_cr(" Added " PTR_FORMAT " Young Regions to CS.",
1.1525 + _inc_cset_size);
1.1526 + gclog_or_tty->print_cr(" (" SIZE_FORMAT " KB left in heap.)",
1.1527 + max_live_bytes/K);
1.1528 + }
1.1529 +
1.1530 + assert(_inc_cset_size == _g1->young_list()->length(), "Invariant");
1.1531 + if (_inc_cset_size > 0) {
1.1532 + assert(_collection_set != NULL, "Invariant");
1.1533 + abandon_collection = false;
1.1534 + }
1.1535
1.1536 double young_end_time_sec = os::elapsedTime();
1.1537 _recorded_young_cset_choice_time_ms =
1.1538 (young_end_time_sec - young_start_time_sec) * 1000.0;
1.1539
1.1540 - non_young_start_time_sec = os::elapsedTime();
1.1541 -
1.1542 - if (_young_cset_length > 0 && _last_young_gc_full) {
1.1543 + // We are doing young collections so reset this.
1.1544 + non_young_start_time_sec = young_end_time_sec;
1.1545 +
1.1546 + // Note we can use either _collection_set_size or
1.1547 + // _young_cset_length here
1.1548 + if (_collection_set_size > 0 && _last_young_gc_full) {
1.1549 // don't bother adding more regions...
1.1550 goto choose_collection_set_end;
1.1551 }
1.1552 @@ -3077,6 +2946,11 @@
1.1553 bool should_continue = true;
1.1554 NumberSeq seq;
1.1555 double avg_prediction = 100000000000000000.0; // something very large
1.1556 +
1.1557 + // Save the current size of the collection set to detect
1.1558 + // if we actually added any old regions.
1.1559 + size_t n_young_regions = _collection_set_size;
1.1560 +
1.1561 do {
1.1562 hr = _collectionSetChooser->getNextMarkedRegion(time_remaining_ms,
1.1563 avg_prediction);
1.1564 @@ -3085,7 +2959,7 @@
1.1565 time_remaining_ms -= predicted_time_ms;
1.1566 predicted_pause_time_ms += predicted_time_ms;
1.1567 add_to_collection_set(hr);
1.1568 - record_cset_region(hr, false);
1.1569 + record_non_young_cset_region(hr);
1.1570 max_live_bytes -= MIN2(hr->max_live_bytes(), max_live_bytes);
1.1571 if (G1PolicyVerbose > 0) {
1.1572 gclog_or_tty->print_cr(" (" SIZE_FORMAT " KB left in heap.)",
1.1573 @@ -3103,9 +2977,17 @@
1.1574 if (!adaptive_young_list_length() &&
1.1575 _collection_set_size < _young_list_fixed_length)
1.1576 _should_revert_to_full_young_gcs = true;
1.1577 +
1.1578 + if (_collection_set_size > n_young_regions) {
1.1579 + // We actually added old regions to the collection set
1.1580 + // so we are not abandoning this collection.
1.1581 + abandon_collection = false;
1.1582 + }
1.1583 }
1.1584
1.1585 choose_collection_set_end:
1.1586 + stop_incremental_cset_building();
1.1587 +
1.1588 count_CS_bytes_used();
1.1589
1.1590 end_recording_regions();
1.1591 @@ -3113,6 +2995,8 @@
1.1592 double non_young_end_time_sec = os::elapsedTime();
1.1593 _recorded_non_young_cset_choice_time_ms =
1.1594 (non_young_end_time_sec - non_young_start_time_sec) * 1000.0;
1.1595 +
1.1596 + return abandon_collection;
1.1597 }
1.1598
1.1599 void G1CollectorPolicy_BestRegionsFirst::record_full_collection_end() {
