Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "precompiled.hpp"

 #include "code/icBuffer.hpp"

 #include "gc_implementation/g1/bufferingOopClosure.hpp"

 #include "gc_implementation/g1/concurrentG1Refine.hpp"

 #include "gc_implementation/g1/concurrentG1RefineThread.hpp"

 #include "gc_implementation/g1/concurrentMarkThread.inline.hpp"

 #include "gc_implementation/g1/g1AllocRegion.inline.hpp"

 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

 #include "gc_implementation/g1/g1CollectorPolicy.hpp"

 #include "gc_implementation/g1/g1ErgoVerbose.hpp"

 #include "gc_implementation/g1/g1EvacFailure.hpp"

 #include "gc_implementation/g1/g1Log.hpp"

 #include "gc_implementation/g1/g1MarkSweep.hpp"

 #include "gc_implementation/g1/g1OopClosures.inline.hpp"

 #include "gc_implementation/g1/g1RemSet.inline.hpp"

 #include "gc_implementation/g1/heapRegion.inline.hpp"

 #include "gc_implementation/g1/heapRegionRemSet.hpp"

 #include "gc_implementation/g1/heapRegionSeq.inline.hpp"

 #include "gc_implementation/g1/vm_operations_g1.hpp"

 #include "gc_implementation/shared/isGCActiveMark.hpp"

 #include "memory/gcLocker.inline.hpp"

 #include "memory/genOopClosures.inline.hpp"

 #include "memory/generationSpec.hpp"

 #include "memory/referenceProcessor.hpp"

 #include "oops/oop.inline.hpp"

 #include "oops/oop.pcgc.inline.hpp"

 #include "runtime/aprofiler.hpp"

 #include "runtime/vmThread.hpp"

 size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0;

 // turn it on so that the contents of the young list (scan-only /

 // to-be-collected) are printed at "strategic" points before / during

 // / after the collection --- this is useful for debugging

 #define YOUNG_LIST_VERBOSE 0

 // CURRENT STATUS

 // This file is under construction.  Search for "FIXME".

 // INVARIANTS/NOTES

 //

 // All allocation activity covered by the G1CollectedHeap interface is

 // serialized by acquiring the HeapLock.  This happens in mem_allocate

 // and allocate_new_tlab, which are the "entry" points to the

 // allocation code from the rest of the JVM.  (Note that this does not

 // apply to TLAB allocation, which is not part of this interface: it

 // is done by clients of this interface.)

 // Notes on implementation of parallelism in different tasks.

 //

 // G1ParVerifyTask uses heap_region_par_iterate_chunked() for parallelism.

 // The number of GC workers is passed to heap_region_par_iterate_chunked().

 // It does use run_task() which sets _n_workers in the task.

 // G1ParTask executes g1_process_strong_roots() ->

 // SharedHeap::process_strong_roots() which calls eventuall to

 // CardTableModRefBS::par_non_clean_card_iterate_work() which uses

 // SequentialSubTasksDone.  SharedHeap::process_strong_roots() also

 // directly uses SubTasksDone (_process_strong_tasks field in SharedHeap).

 //

 // Local to this file.

 class RefineCardTableEntryClosure: public CardTableEntryClosure {

   SuspendibleThreadSet* _sts;

   G1RemSet* _g1rs;

   ConcurrentG1Refine* _cg1r;

   bool _concurrent;

 public:

   RefineCardTableEntryClosure(SuspendibleThreadSet* sts,

                               G1RemSet* g1rs,

                               ConcurrentG1Refine* cg1r) :

     _sts(sts), _g1rs(g1rs), _cg1r(cg1r), _concurrent(true)

   {}

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     bool oops_into_cset = _g1rs->concurrentRefineOneCard(card_ptr, worker_i, false);

     // This path is executed by the concurrent refine or mutator threads,

     // concurrently, and so we do not care if card_ptr contains references

     // that point into the collection set.

     assert(!oops_into_cset, "should be");

     if (_concurrent && _sts->should_yield()) {

       // Caller will actually yield.

       return false;

     }

     // Otherwise, we finished successfully; return true.

     return true;

   }

   void set_concurrent(bool b) { _concurrent = b; }

 };

 class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure {

   int _calls;

   G1CollectedHeap* _g1h;

   CardTableModRefBS* _ctbs;

   int _histo[256];

 public:

   ClearLoggedCardTableEntryClosure() :

     _calls(0)

   {

     _g1h = G1CollectedHeap::heap();

     _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

     for (int i = 0; i < 256; i++) _histo[i] = 0;

   }

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

       _calls++;

       unsigned char* ujb = (unsigned char*)card_ptr;

       int ind = (int)(*ujb);

       _histo[ind]++;

       *card_ptr = -1;

     }

     return true;

   }

   int calls() { return _calls; }

   void print_histo() {

     gclog_or_tty->print_cr("Card table value histogram:");

     for (int i = 0; i < 256; i++) {

       if (_histo[i] != 0) {

         gclog_or_tty->print_cr("  %d: %d", i, _histo[i]);

       }

     }

   }

 };

 class RedirtyLoggedCardTableEntryClosure: public CardTableEntryClosure {

   int _calls;

   G1CollectedHeap* _g1h;

   CardTableModRefBS* _ctbs;

 public:

   RedirtyLoggedCardTableEntryClosure() :

     _calls(0)

   {

     _g1h = G1CollectedHeap::heap();

     _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

   }

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

       _calls++;

       *card_ptr = 0;

     }

     return true;

   }

   int calls() { return _calls; }

 };

 class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure {

 public:

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     *card_ptr = CardTableModRefBS::dirty_card_val();

     return true;

   }

 };

 YoungList::YoungList(G1CollectedHeap* g1h) :

     _g1h(g1h), _head(NULL), _length(0), _last_sampled_rs_lengths(0),

     _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0) {

   guarantee(check_list_empty(false), "just making sure...");

 }

 void YoungList::push_region(HeapRegion *hr) {

   assert(!hr->is_young(), "should not already be young");

   assert(hr->get_next_young_region() == NULL, "cause it should!");

   hr->set_next_young_region(_head);

   _head = hr;

   _g1h->g1_policy()->set_region_eden(hr, (int) _length);

   ++_length;

 }

 void YoungList::add_survivor_region(HeapRegion* hr) {

   assert(hr->is_survivor(), "should be flagged as survivor region");

   assert(hr->get_next_young_region() == NULL, "cause it should!");

   hr->set_next_young_region(_survivor_head);

   if (_survivor_head == NULL) {

     _survivor_tail = hr;

   }

   _survivor_head = hr;

   ++_survivor_length;

 }

 void YoungList::empty_list(HeapRegion* list) {

   while (list != NULL) {

     HeapRegion* next = list->get_next_young_region();

     list->set_next_young_region(NULL);

     list->uninstall_surv_rate_group();

     list->set_not_young();

     list = next;

   }

 }

 void YoungList::empty_list() {

   assert(check_list_well_formed(), "young list should be well formed");

   empty_list(_head);

   _head = NULL;

   _length = 0;

   empty_list(_survivor_head);

   _survivor_head = NULL;

   _survivor_tail = NULL;

   _survivor_length = 0;

   _last_sampled_rs_lengths = 0;

   assert(check_list_empty(false), "just making sure...");

 }

 bool YoungList::check_list_well_formed() {

   bool ret = true;

   uint length = 0;

   HeapRegion* curr = _head;

   HeapRegion* last = NULL;

   while (curr != NULL) {

     if (!curr->is_young()) {

       gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" "

                              "incorrectly tagged (y: %d, surv: %d)",

                              curr->bottom(), curr->end(),

                              curr->is_young(), curr->is_survivor());

       ret = false;

     }

     ++length;

     last = curr;

     curr = curr->get_next_young_region();

   }

   ret = ret && (length == _length);

   if (!ret) {

     gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!");

     gclog_or_tty->print_cr("###   list has %u entries, _length is %u",

                            length, _length);

   }

   return ret;

 }

 bool YoungList::check_list_empty(bool check_sample) {

   bool ret = true;

   if (_length != 0) {

     gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %u",

                   _length);

     ret = false;

   }

   if (check_sample && _last_sampled_rs_lengths != 0) {

     gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths");

     ret = false;

   }

   if (_head != NULL) {

     gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head");

     ret = false;

   }

   if (!ret) {

     gclog_or_tty->print_cr("### YOUNG LIST does not seem empty");

   }

   return ret;

 }

 void

 YoungList::rs_length_sampling_init() {

   _sampled_rs_lengths = 0;

   _curr               = _head;

 }

 bool

 YoungList::rs_length_sampling_more() {

   return _curr != NULL;

 }

 void

 YoungList::rs_length_sampling_next() {

   assert( _curr != NULL, "invariant" );

   size_t rs_length = _curr->rem_set()->occupied();

   _sampled_rs_lengths += rs_length;

   // The current region may not yet have been added to the

   // incremental collection set (it gets added when it is

   // retired as the current allocation region).

   if (_curr->in_collection_set()) {

     // Update the collection set policy information for this region

     _g1h->g1_policy()->update_incremental_cset_info(_curr, rs_length);

   }

   _curr = _curr->get_next_young_region();

   if (_curr == NULL) {

     _last_sampled_rs_lengths = _sampled_rs_lengths;

     // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths);

   }

 }

 void

 YoungList::reset_auxilary_lists() {

   guarantee( is_empty(), "young list should be empty" );

   assert(check_list_well_formed(), "young list should be well formed");

   // Add survivor regions to SurvRateGroup.

   _g1h->g1_policy()->note_start_adding_survivor_regions();

   _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */);

   int young_index_in_cset = 0;

   for (HeapRegion* curr = _survivor_head;

        curr != NULL;

        curr = curr->get_next_young_region()) {

     _g1h->g1_policy()->set_region_survivor(curr, young_index_in_cset);

     // The region is a non-empty survivor so let's add it to

     // the incremental collection set for the next evacuation

     // pause.

     _g1h->g1_policy()->add_region_to_incremental_cset_rhs(curr);

     young_index_in_cset += 1;

   }

   assert((uint) young_index_in_cset == _survivor_length, "post-condition");

   _g1h->g1_policy()->note_stop_adding_survivor_regions();

   _head   = _survivor_head;

   _length = _survivor_length;

   if (_survivor_head != NULL) {

     assert(_survivor_tail != NULL, "cause it shouldn't be");

     assert(_survivor_length > 0, "invariant");

     _survivor_tail->set_next_young_region(NULL);

   }

   // Don't clear the survivor list handles until the start of

   // the next evacuation pause - we need it in order to re-tag

   // the survivor regions from this evacuation pause as 'young'

   // at the start of the next.

   _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */);

   assert(check_list_well_formed(), "young list should be well formed");

 }

 void YoungList::print() {

   HeapRegion* lists[] = {_head,   _survivor_head};

   const char* names[] = {"YOUNG", "SURVIVOR"};

   for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) {

     gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]);

     HeapRegion *curr = lists[list];

     if (curr == NULL)

       gclog_or_tty->print_cr("  empty");

     while (curr != NULL) {

       gclog_or_tty->print_cr("  "HR_FORMAT", P: "PTR_FORMAT "N: "PTR_FORMAT", age: %4d",

                              HR_FORMAT_PARAMS(curr),

                              curr->prev_top_at_mark_start(),

                              curr->next_top_at_mark_start(),

                              curr->age_in_surv_rate_group_cond());

       curr = curr->get_next_young_region();

     }

   }

   gclog_or_tty->print_cr("");

 }

 void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr)

 {

   // Claim the right to put the region on the dirty cards region list

   // by installing a self pointer.

   HeapRegion* next = hr->get_next_dirty_cards_region();

   if (next == NULL) {

     HeapRegion* res = (HeapRegion*)

       Atomic::cmpxchg_ptr(hr, hr->next_dirty_cards_region_addr(),

                           NULL);

     if (res == NULL) {

       HeapRegion* head;

       do {

         // Put the region to the dirty cards region list.

         head = _dirty_cards_region_list;

         next = (HeapRegion*)

           Atomic::cmpxchg_ptr(hr, &_dirty_cards_region_list, head);

         if (next == head) {

           assert(hr->get_next_dirty_cards_region() == hr,

                  "hr->get_next_dirty_cards_region() != hr");

           if (next == NULL) {

             // The last region in the list points to itself.

             hr->set_next_dirty_cards_region(hr);

           } else {

             hr->set_next_dirty_cards_region(next);

           }

         }

       } while (next != head);

     }

   }

 }

 HeapRegion* G1CollectedHeap::pop_dirty_cards_region()

 {

   HeapRegion* head;

   HeapRegion* hr;

   do {

     head = _dirty_cards_region_list;

     if (head == NULL) {

       return NULL;

     }

     HeapRegion* new_head = head->get_next_dirty_cards_region();

     if (head == new_head) {

       // The last region.

       new_head = NULL;

     }

     hr = (HeapRegion*)Atomic::cmpxchg_ptr(new_head, &_dirty_cards_region_list,

                                           head);

   } while (hr != head);

   assert(hr != NULL, "invariant");

   hr->set_next_dirty_cards_region(NULL);

   return hr;

 }

 void G1CollectedHeap::stop_conc_gc_threads() {

   _cg1r->stop();

   _cmThread->stop();

 }

 #ifdef ASSERT

 // A region is added to the collection set as it is retired

 // so an address p can point to a region which will be in the

 // collection set but has not yet been retired.  This method

 // therefore is only accurate during a GC pause after all

 // regions have been retired.  It is used for debugging

 // to check if an nmethod has references to objects that can

 // be move during a partial collection.  Though it can be

 // inaccurate, it is sufficient for G1 because the conservative

 // implementation of is_scavengable() for G1 will indicate that

 // all nmethods must be scanned during a partial collection.

 bool G1CollectedHeap::is_in_partial_collection(const void* p) {

   HeapRegion* hr = heap_region_containing(p);

   return hr != NULL && hr->in_collection_set();

 }

 #endif

 // Returns true if the reference points to an object that

 // can move in an incremental collecction.

 bool G1CollectedHeap::is_scavengable(const void* p) {

   G1CollectedHeap* g1h = G1CollectedHeap::heap();

   G1CollectorPolicy* g1p = g1h->g1_policy();

   HeapRegion* hr = heap_region_containing(p);

   if (hr == NULL) {

      // perm gen (or null)

      return false;

   } else {

     return !hr->isHumongous();

   }

 }

 void G1CollectedHeap::check_ct_logs_at_safepoint() {

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set();

   // Count the dirty cards at the start.

   CountNonCleanMemRegionClosure count1(this);

   ct_bs->mod_card_iterate(&count1);

   int orig_count = count1.n();

   // First clear the logged cards.

   ClearLoggedCardTableEntryClosure clear;

   dcqs.set_closure(&clear);

   dcqs.apply_closure_to_all_completed_buffers();

   dcqs.iterate_closure_all_threads(false);

   clear.print_histo();

   // Now ensure that there's no dirty cards.

   CountNonCleanMemRegionClosure count2(this);

   ct_bs->mod_card_iterate(&count2);

   if (count2.n() != 0) {

     gclog_or_tty->print_cr("Card table has %d entries; %d originally",

                            count2.n(), orig_count);

   }

   guarantee(count2.n() == 0, "Card table should be clean.");

   RedirtyLoggedCardTableEntryClosure redirty;

   JavaThread::dirty_card_queue_set().set_closure(&redirty);

   dcqs.apply_closure_to_all_completed_buffers();

   dcqs.iterate_closure_all_threads(false);

   gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.",

                          clear.calls(), orig_count);

   guarantee(redirty.calls() == clear.calls(),

             "Or else mechanism is broken.");

   CountNonCleanMemRegionClosure count3(this);

   ct_bs->mod_card_iterate(&count3);

   if (count3.n() != orig_count) {

     gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.",

                            orig_count, count3.n());

     guarantee(count3.n() >= orig_count, "Should have restored them all.");

   }

   JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

 }

 // Private class members.

 G1CollectedHeap* G1CollectedHeap::_g1h;

 // Private methods.

 HeapRegion*

 G1CollectedHeap::new_region_try_secondary_free_list() {

   MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);

   while (!_secondary_free_list.is_empty() || free_regions_coming()) {

     if (!_secondary_free_list.is_empty()) {

       if (G1ConcRegionFreeingVerbose) {

         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "

                                "secondary_free_list has %u entries",

                                _secondary_free_list.length());

       }

       // It looks as if there are free regions available on the

       // secondary_free_list. Let's move them to the free_list and try

       // again to allocate from it.

       append_secondary_free_list();

       assert(!_free_list.is_empty(), "if the secondary_free_list was not "

              "empty we should have moved at least one entry to the free_list");

       HeapRegion* res = _free_list.remove_head();

       if (G1ConcRegionFreeingVerbose) {

         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "

                                "allocated "HR_FORMAT" from secondary_free_list",

                                HR_FORMAT_PARAMS(res));

       }

       return res;

     }

     // Wait here until we get notifed either when (a) there are no

     // more free regions coming or (b) some regions have been moved on

     // the secondary_free_list.

     SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag);

   }

   if (G1ConcRegionFreeingVerbose) {

     gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "

                            "could not allocate from secondary_free_list");

   }

   return NULL;

 }

 HeapRegion* G1CollectedHeap::new_region(size_t word_size, bool do_expand) {

   assert(!isHumongous(word_size) || word_size <= HeapRegion::GrainWords,

          "the only time we use this to allocate a humongous region is "

          "when we are allocating a single humongous region");

   HeapRegion* res;

   if (G1StressConcRegionFreeing) {

     if (!_secondary_free_list.is_empty()) {

       if (G1ConcRegionFreeingVerbose) {

         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "

                                "forced to look at the secondary_free_list");

       }

       res = new_region_try_secondary_free_list();

       if (res != NULL) {

         return res;

       }

     }

   }

   res = _free_list.remove_head_or_null();

   if (res == NULL) {

     if (G1ConcRegionFreeingVerbose) {

       gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "

                              "res == NULL, trying the secondary_free_list");

     }

     res = new_region_try_secondary_free_list();

   }

   if (res == NULL && do_expand && _expand_heap_after_alloc_failure) {

     // Currently, only attempts to allocate GC alloc regions set

     // do_expand to true. So, we should only reach here during a

     // safepoint. If this assumption changes we might have to

     // reconsider the use of _expand_heap_after_alloc_failure.

     assert(SafepointSynchronize::is_at_safepoint(), "invariant");

     ergo_verbose1(ErgoHeapSizing,

                   "attempt heap expansion",

                   ergo_format_reason("region allocation request failed")

                   ergo_format_byte("allocation request"),

                   word_size * HeapWordSize);

     if (expand(word_size * HeapWordSize)) {

       // Given that expand() succeeded in expanding the heap, and we

       // always expand the heap by an amount aligned to the heap

       // region size, the free list should in theory not be empty. So

       // it would probably be OK to use remove_head(). But the extra

       // check for NULL is unlikely to be a performance issue here (we

       // just expanded the heap!) so let's just be conservative and

       // use remove_head_or_null().

       res = _free_list.remove_head_or_null();

     } else {

       _expand_heap_after_alloc_failure = false;

     }

   }

   return res;

 }

 uint G1CollectedHeap::humongous_obj_allocate_find_first(uint num_regions,

                                                         size_t word_size) {

   assert(isHumongous(word_size), "word_size should be humongous");

   assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");

   uint first = G1_NULL_HRS_INDEX;

   if (num_regions == 1) {

     // Only one region to allocate, no need to go through the slower

     // path. The caller will attempt the expasion if this fails, so

     // let's not try to expand here too.

     HeapRegion* hr = new_region(word_size, false /* do_expand */);

     if (hr != NULL) {

       first = hr->hrs_index();

     } else {

       first = G1_NULL_HRS_INDEX;

     }

   } else {

     // We can't allocate humongous regions while cleanupComplete() is

     // running, since some of the regions we find to be empty might not

     // yet be added to the free list and it is not straightforward to

     // know which list they are on so that we can remove them. Note

     // that we only need to do this if we need to allocate more than

     // one region to satisfy the current humongous allocation

     // request. If we are only allocating one region we use the common

     // region allocation code (see above).

     wait_while_free_regions_coming();

     append_secondary_free_list_if_not_empty_with_lock();

     if (free_regions() >= num_regions) {

       first = _hrs.find_contiguous(num_regions);

       if (first != G1_NULL_HRS_INDEX) {

         for (uint i = first; i < first + num_regions; ++i) {

           HeapRegion* hr = region_at(i);

           assert(hr->is_empty(), "sanity");

           assert(is_on_master_free_list(hr), "sanity");

           hr->set_pending_removal(true);

         }

         _free_list.remove_all_pending(num_regions);

       }

     }

   }

   return first;

 }

 HeapWord*

 G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first,

                                                            uint num_regions,

                                                            size_t word_size) {

   assert(first != G1_NULL_HRS_INDEX, "pre-condition");

   assert(isHumongous(word_size), "word_size should be humongous");

   assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");

   // Index of last region in the series + 1.

   uint last = first + num_regions;

   // We need to initialize the region(s) we just discovered. This is

   // a bit tricky given that it can happen concurrently with

   // refinement threads refining cards on these regions and

   // potentially wanting to refine the BOT as they are scanning

   // those cards (this can happen shortly after a cleanup; see CR

   // 6991377). So we have to set up the region(s) carefully and in

   // a specific order.

   // The word size sum of all the regions we will allocate.

   size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;

   assert(word_size <= word_size_sum, "sanity");

   // This will be the "starts humongous" region.

   HeapRegion* first_hr = region_at(first);

   // The header of the new object will be placed at the bottom of

   // the first region.

   HeapWord* new_obj = first_hr->bottom();

   // This will be the new end of the first region in the series that

   // should also match the end of the last region in the seriers.

   HeapWord* new_end = new_obj + word_size_sum;

   // This will be the new top of the first region that will reflect

   // this allocation.

   HeapWord* new_top = new_obj + word_size;

   // First, we need to zero the header of the space that we will be

   // allocating. When we update top further down, some refinement

   // threads might try to scan the region. By zeroing the header we

   // ensure that any thread that will try to scan the region will

   // come across the zero klass word and bail out.

   //

   // NOTE: It would not have been correct to have used

   // CollectedHeap::fill_with_object() and make the space look like

   // an int array. The thread that is doing the allocation will

   // later update the object header to a potentially different array

   // type and, for a very short period of time, the klass and length

   // fields will be inconsistent. This could cause a refinement

   // thread to calculate the object size incorrectly.

   Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);

   // We will set up the first region as "starts humongous". This

   // will also update the BOT covering all the regions to reflect

   // that there is a single object that starts at the bottom of the

   // first region.

   first_hr->set_startsHumongous(new_top, new_end);

   // Then, if there are any, we will set up the "continues

   // humongous" regions.

   HeapRegion* hr = NULL;

   for (uint i = first + 1; i < last; ++i) {

     hr = region_at(i);

     hr->set_continuesHumongous(first_hr);

   }

   // If we have "continues humongous" regions (hr != NULL), then the

   // end of the last one should match new_end.

   assert(hr == NULL || hr->end() == new_end, "sanity");

   // Up to this point no concurrent thread would have been able to

   // do any scanning on any region in this series. All the top

   // fields still point to bottom, so the intersection between

   // [bottom,top] and [card_start,card_end] will be empty. Before we

   // update the top fields, we'll do a storestore to make sure that

   // no thread sees the update to top before the zeroing of the

   // object header and the BOT initialization.

   OrderAccess::storestore();

   // Now that the BOT and the object header have been initialized,

   // we can update top of the "starts humongous" region.

   assert(first_hr->bottom() < new_top && new_top <= first_hr->end(),

          "new_top should be in this region");

   first_hr->set_top(new_top);

   if (_hr_printer.is_active()) {

     HeapWord* bottom = first_hr->bottom();

     HeapWord* end = first_hr->orig_end();

     if ((first + 1) == last) {

       // the series has a single humongous region

       _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, new_top);

     } else {

       // the series has more than one humongous regions

       _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, end);

     }

   }

   // Now, we will update the top fields of the "continues humongous"

   // regions. The reason we need to do this is that, otherwise,

   // these regions would look empty and this will confuse parts of

   // G1. For example, the code that looks for a consecutive number

   // of empty regions will consider them empty and try to

   // re-allocate them. We can extend is_empty() to also include

   // !continuesHumongous(), but it is easier to just update the top

   // fields here. The way we set top for all regions (i.e., top ==

   // end for all regions but the last one, top == new_top for the

   // last one) is actually used when we will free up the humongous

   // region in free_humongous_region().

   hr = NULL;

   for (uint i = first + 1; i < last; ++i) {

     hr = region_at(i);

     if ((i + 1) == last) {

       // last continues humongous region

       assert(hr->bottom() < new_top && new_top <= hr->end(),

              "new_top should fall on this region");

       hr->set_top(new_top);

       _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, new_top);

     } else {

       // not last one

       assert(new_top > hr->end(), "new_top should be above this region");

       hr->set_top(hr->end());

       _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end());

     }

   }

   // If we have continues humongous regions (hr != NULL), then the

   // end of the last one should match new_end and its top should

   // match new_top.

   assert(hr == NULL ||

          (hr->end() == new_end && hr->top() == new_top), "sanity");

   assert(first_hr->used() == word_size * HeapWordSize, "invariant");

   _summary_bytes_used += first_hr->used();

   _humongous_set.add(first_hr);

   return new_obj;

 }

 // If could fit into free regions w/o expansion, try.

 // Otherwise, if can expand, do so.

 // Otherwise, if using ex regions might help, try with ex given back.

 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size) {

   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);

   verify_region_sets_optional();

   size_t word_size_rounded = round_to(word_size, HeapRegion::GrainWords);

   uint num_regions = (uint) (word_size_rounded / HeapRegion::GrainWords);

   uint x_num = expansion_regions();

   uint fs = _hrs.free_suffix();

   uint first = humongous_obj_allocate_find_first(num_regions, word_size);

   if (first == G1_NULL_HRS_INDEX) {

     // The only thing we can do now is attempt expansion.

     if (fs + x_num >= num_regions) {

       // If the number of regions we're trying to allocate for this

       // object is at most the number of regions in the free suffix,

       // then the call to humongous_obj_allocate_find_first() above

       // should have succeeded and we wouldn't be here.

       //

       // We should only be trying to expand when the free suffix is

       // not sufficient for the object _and_ we have some expansion

       // room available.

       assert(num_regions > fs, "earlier allocation should have succeeded");

       ergo_verbose1(ErgoHeapSizing,

                     "attempt heap expansion",

                     ergo_format_reason("humongous allocation request failed")

                     ergo_format_byte("allocation request"),

                     word_size * HeapWordSize);

       if (expand((num_regions - fs) * HeapRegion::GrainBytes)) {

         // Even though the heap was expanded, it might not have

         // reached the desired size. So, we cannot assume that the

         // allocation will succeed.

         first = humongous_obj_allocate_find_first(num_regions, word_size);

       }

     }

   }

   HeapWord* result = NULL;

   if (first != G1_NULL_HRS_INDEX) {

     result =

       humongous_obj_allocate_initialize_regions(first, num_regions, word_size);

     assert(result != NULL, "it should always return a valid result");

     // A successful humongous object allocation changes the used space

     // information of the old generation so we need to recalculate the

     // sizes and update the jstat counters here.

     g1mm()->update_sizes();

   }

   verify_region_sets_optional();

   return result;

 }

 HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) {

   assert_heap_not_locked_and_not_at_safepoint();

   assert(!isHumongous(word_size), "we do not allow humongous TLABs");

   unsigned int dummy_gc_count_before;

   return attempt_allocation(word_size, &dummy_gc_count_before);

 }

 HeapWord*

 G1CollectedHeap::mem_allocate(size_t word_size,

                               bool*  gc_overhead_limit_was_exceeded) {

   assert_heap_not_locked_and_not_at_safepoint();

   // Loop until the allocation is satisified, or unsatisfied after GC.

   for (int try_count = 1; /* we'll return */; try_count += 1) {

     unsigned int gc_count_before;

     HeapWord* result = NULL;

     if (!isHumongous(word_size)) {

       result = attempt_allocation(word_size, &gc_count_before);

     } else {

       result = attempt_allocation_humongous(word_size, &gc_count_before);

     }

     if (result != NULL) {

       return result;

     }

     // Create the garbage collection operation...

     VM_G1CollectForAllocation op(gc_count_before, word_size);

     // ...and get the VM thread to execute it.

     VMThread::execute(&op);

     if (op.prologue_succeeded() && op.pause_succeeded()) {

       // If the operation was successful we'll return the result even

       // if it is NULL. If the allocation attempt failed immediately

       // after a Full GC, it's unlikely we'll be able to allocate now.

       HeapWord* result = op.result();

       if (result != NULL && !isHumongous(word_size)) {

         // Allocations that take place on VM operations do not do any

         // card dirtying and we have to do it here. We only have to do

         // this for non-humongous allocations, though.

         dirty_young_block(result, word_size);

       }

       return result;

     } else {

       assert(op.result() == NULL,

              "the result should be NULL if the VM op did not succeed");

     }

     // Give a warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::mem_allocate retries %d times", try_count);

     }

   }

   ShouldNotReachHere();

   return NULL;

 }

 HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size,

                                            unsigned int *gc_count_before_ret) {

   // Make sure you read the note in attempt_allocation_humongous().

   assert_heap_not_locked_and_not_at_safepoint();

   assert(!isHumongous(word_size), "attempt_allocation_slow() should not "

          "be called for humongous allocation requests");

   // We should only get here after the first-level allocation attempt

   // (attempt_allocation()) failed to allocate.

   // We will loop until a) we manage to successfully perform the

   // allocation or b) we successfully schedule a collection which

   // fails to perform the allocation. b) is the only case when we'll

   // return NULL.

   HeapWord* result = NULL;

   for (int try_count = 1; /* we'll return */; try_count += 1) {

     bool should_try_gc;

     unsigned int gc_count_before;

     {

       MutexLockerEx x(Heap_lock);

       result = _mutator_alloc_region.attempt_allocation_locked(word_size,

                                                       false /* bot_updates */);

       if (result != NULL) {

         return result;

       }

       // If we reach here, attempt_allocation_locked() above failed to

       // allocate a new region. So the mutator alloc region should be NULL.

       assert(_mutator_alloc_region.get() == NULL, "only way to get here");

       if (GC_locker::is_active_and_needs_gc()) {

         if (g1_policy()->can_expand_young_list()) {

           // No need for an ergo verbose message here,

           // can_expand_young_list() does this when it returns true.

           result = _mutator_alloc_region.attempt_allocation_force(word_size,

                                                       false /* bot_updates */);

           if (result != NULL) {

             return result;

           }

         }

         should_try_gc = false;

       } else {

         // Read the GC count while still holding the Heap_lock.

         gc_count_before = total_collections();

         should_try_gc = true;

       }

     }

     if (should_try_gc) {

       bool succeeded;

       result = do_collection_pause(word_size, gc_count_before, &succeeded);

       if (result != NULL) {

         assert(succeeded, "only way to get back a non-NULL result");

         return result;

       }

       if (succeeded) {

         // If we get here we successfully scheduled a collection which

         // failed to allocate. No point in trying to allocate

         // further. We'll just return NULL.

         MutexLockerEx x(Heap_lock);

         *gc_count_before_ret = total_collections();

         return NULL;

       }

     } else {

       GC_locker::stall_until_clear();

     }

     // We can reach here if we were unsuccessul in scheduling a

     // collection (because another thread beat us to it) or if we were

     // stalled due to the GC locker. In either can we should retry the

     // allocation attempt in case another thread successfully

     // performed a collection and reclaimed enough space. We do the

     // first attempt (without holding the Heap_lock) here and the

     // follow-on attempt will be at the start of the next loop

     // iteration (after taking the Heap_lock).

     result = _mutator_alloc_region.attempt_allocation(word_size,

                                                       false /* bot_updates */);

     if (result != NULL) {

       return result;

     }

     // Give a warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::attempt_allocation_slow() "

               "retries %d times", try_count);

     }

   }

   ShouldNotReachHere();

   return NULL;

 }

 HeapWord* G1CollectedHeap::attempt_allocation_humongous(size_t word_size,

                                           unsigned int * gc_count_before_ret) {

   // The structure of this method has a lot of similarities to

   // attempt_allocation_slow(). The reason these two were not merged

   // into a single one is that such a method would require several "if

   // allocation is not humongous do this, otherwise do that"

   // conditional paths which would obscure its flow. In fact, an early

   // version of this code did use a unified method which was harder to

   // follow and, as a result, it had subtle bugs that were hard to

   // track down. So keeping these two methods separate allows each to

   // be more readable. It will be good to keep these two in sync as

   // much as possible.

   assert_heap_not_locked_and_not_at_safepoint();

   assert(isHumongous(word_size), "attempt_allocation_humongous() "

          "should only be called for humongous allocations");

   // Humongous objects can exhaust the heap quickly, so we should check if we

   // need to start a marking cycle at each humongous object allocation. We do

   // the check before we do the actual allocation. The reason for doing it

   // before the allocation is that we avoid having to keep track of the newly

   // allocated memory while we do a GC.

   if (g1_policy()->need_to_start_conc_mark("concurrent humongous allocation",

                                            word_size)) {

     collect(GCCause::_g1_humongous_allocation);

   }

   // We will loop until a) we manage to successfully perform the

   // allocation or b) we successfully schedule a collection which

   // fails to perform the allocation. b) is the only case when we'll

   // return NULL.

   HeapWord* result = NULL;

   for (int try_count = 1; /* we'll return */; try_count += 1) {

     bool should_try_gc;

     unsigned int gc_count_before;

     {

       MutexLockerEx x(Heap_lock);

       // Given that humongous objects are not allocated in young

       // regions, we'll first try to do the allocation without doing a

       // collection hoping that there's enough space in the heap.

       result = humongous_obj_allocate(word_size);

       if (result != NULL) {

         return result;

       }

       if (GC_locker::is_active_and_needs_gc()) {

         should_try_gc = false;

       } else {

         // Read the GC count while still holding the Heap_lock.

         gc_count_before = total_collections();

         should_try_gc = true;

       }

     }

     if (should_try_gc) {

       // If we failed to allocate the humongous object, we should try to

       // do a collection pause (if we're allowed) in case it reclaims

       // enough space for the allocation to succeed after the pause.

       bool succeeded;

       result = do_collection_pause(word_size, gc_count_before, &succeeded);

       if (result != NULL) {

         assert(succeeded, "only way to get back a non-NULL result");

         return result;

       }

       if (succeeded) {

         // If we get here we successfully scheduled a collection which

         // failed to allocate. No point in trying to allocate

         // further. We'll just return NULL.

         MutexLockerEx x(Heap_lock);

         *gc_count_before_ret = total_collections();

         return NULL;

       }

     } else {

       GC_locker::stall_until_clear();

     }

     // We can reach here if we were unsuccessul in scheduling a

     // collection (because another thread beat us to it) or if we were

     // stalled due to the GC locker. In either can we should retry the

     // allocation attempt in case another thread successfully

     // performed a collection and reclaimed enough space.  Give a

     // warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::attempt_allocation_humongous() "

               "retries %d times", try_count);

     }

   }

   ShouldNotReachHere();

   return NULL;

 }

 HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size,

                                        bool expect_null_mutator_alloc_region) {

   assert_at_safepoint(true /* should_be_vm_thread */);

   assert(_mutator_alloc_region.get() == NULL ||

                                              !expect_null_mutator_alloc_region,

          "the current alloc region was unexpectedly found to be non-NULL");

   if (!isHumongous(word_size)) {

     return _mutator_alloc_region.attempt_allocation_locked(word_size,

                                                       false /* bot_updates */);

   } else {

     HeapWord* result = humongous_obj_allocate(word_size);

     if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) {

       g1_policy()->set_initiate_conc_mark_if_possible();

     }

     return result;

   }

   ShouldNotReachHere();

 }

 class PostMCRemSetClearClosure: public HeapRegionClosure {

   ModRefBarrierSet* _mr_bs;

 public:

   PostMCRemSetClearClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}

   bool doHeapRegion(HeapRegion* r) {

     r->reset_gc_time_stamp();

     if (r->continuesHumongous())

       return false;

     HeapRegionRemSet* hrrs = r->rem_set();

     if (hrrs != NULL) hrrs->clear();

     // You might think here that we could clear just the cards

     // corresponding to the used region.  But no: if we leave a dirty card

     // in a region we might allocate into, then it would prevent that card

     // from being enqueued, and cause it to be missed.

     // Re: the performance cost: we shouldn't be doing full GC anyway!

     _mr_bs->clear(MemRegion(r->bottom(), r->end()));

     return false;

   }

 };

 class PostMCRemSetInvalidateClosure: public HeapRegionClosure {

   ModRefBarrierSet* _mr_bs;

 public:

   PostMCRemSetInvalidateClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}

   bool doHeapRegion(HeapRegion* r) {

     if (r->continuesHumongous()) return false;

     if (r->used_region().word_size() != 0) {

       _mr_bs->invalidate(r->used_region(), true /*whole heap*/);

     }

     return false;

   }

 };

 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {

   G1CollectedHeap*   _g1h;

   UpdateRSOopClosure _cl;

   int                _worker_i;

 public:

   RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, int worker_i = 0) :

     _cl(g1->g1_rem_set(), worker_i),

     _worker_i(worker_i),

     _g1h(g1)

   { }

   bool doHeapRegion(HeapRegion* r) {

     if (!r->continuesHumongous()) {

       _cl.set_from(r);

       r->oop_iterate(&_cl);

     }

     return false;

   }

 };

 class ParRebuildRSTask: public AbstractGangTask {

   G1CollectedHeap* _g1;

 public:

   ParRebuildRSTask(G1CollectedHeap* g1)

     : AbstractGangTask("ParRebuildRSTask"),

       _g1(g1)

   { }

   void work(uint worker_id) {

     RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id);

     _g1->heap_region_par_iterate_chunked(&rebuild_rs, worker_id,

                                           _g1->workers()->active_workers(),

                                          HeapRegion::RebuildRSClaimValue);

   }

 };

 class PostCompactionPrinterClosure: public HeapRegionClosure {

 private:

   G1HRPrinter* _hr_printer;

 public:

   bool doHeapRegion(HeapRegion* hr) {

     assert(!hr->is_young(), "not expecting to find young regions");

     // We only generate output for non-empty regions.

     if (!hr->is_empty()) {

       if (!hr->isHumongous()) {

         _hr_printer->post_compaction(hr, G1HRPrinter::Old);

       } else if (hr->startsHumongous()) {

         if (hr->capacity() == HeapRegion::GrainBytes) {

           // single humongous region

           _hr_printer->post_compaction(hr, G1HRPrinter::SingleHumongous);

         } else {

           _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous);

         }

       } else {

         assert(hr->continuesHumongous(), "only way to get here");

         _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous);

       }

     }

     return false;

   }

   PostCompactionPrinterClosure(G1HRPrinter* hr_printer)

     : _hr_printer(hr_printer) { }

 };

 bool G1CollectedHeap::do_collection(bool explicit_gc,

                                     bool clear_all_soft_refs,

                                     size_t word_size) {

   assert_at_safepoint(true /* should_be_vm_thread */);

   if (GC_locker::check_active_before_gc()) {

     return false;

   }

   SvcGCMarker sgcm(SvcGCMarker::FULL);

   ResourceMark rm;

   print_heap_before_gc();

   HRSPhaseSetter x(HRSPhaseFullGC);

   verify_region_sets_optional();

   const bool do_clear_all_soft_refs = clear_all_soft_refs ||

                            collector_policy()->should_clear_all_soft_refs();

   ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy());

   {

     IsGCActiveMark x;

     // Timing

     assert(gc_cause() != GCCause::_java_lang_system_gc || explicit_gc, "invariant");

     gclog_or_tty->date_stamp(G1Log::fine() && PrintGCDateStamps);

     TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty);

     char verbose_str[128];

     sprintf(verbose_str, "Full GC (%s)", GCCause::to_string(gc_cause()));

     TraceTime t(verbose_str, G1Log::fine(), true, gclog_or_tty);

     TraceCollectorStats tcs(g1mm()->full_collection_counters());

     TraceMemoryManagerStats tms(true /* fullGC */, gc_cause());

     double start = os::elapsedTime();

     g1_policy()->record_full_collection_start();

     // Note: When we have a more flexible GC logging framework that

     // allows us to add optional attributes to a GC log record we

     // could consider timing and reporting how long we wait in the

     // following two methods.

     wait_while_free_regions_coming();

     // If we start the compaction before the CM threads finish

     // scanning the root regions we might trip them over as we'll

     // be moving objects / updating references. So let's wait until

     // they are done. By telling them to abort, they should complete

     // early.

     _cm->root_regions()->abort();

     _cm->root_regions()->wait_until_scan_finished();

     append_secondary_free_list_if_not_empty_with_lock();

     gc_prologue(true);

     increment_total_collections(true /* full gc */);

     size_t g1h_prev_used = used();

     assert(used() == recalculate_used(), "Should be equal");

     if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       gclog_or_tty->print(" VerifyBeforeGC:");

       prepare_for_verify();

       Universe::verify(/* silent      */ false,

                        /* option      */ VerifyOption_G1UsePrevMarking);

     }

     pre_full_gc_dump();

     COMPILER2_PRESENT(DerivedPointerTable::clear());

     // Disable discovery and empty the discovered lists

     // for the CM ref processor.

     ref_processor_cm()->disable_discovery();

     ref_processor_cm()->abandon_partial_discovery();

     ref_processor_cm()->verify_no_references_recorded();

     // Abandon current iterations of concurrent marking and concurrent

     // refinement, if any are in progress. We have to do this before

     // wait_until_scan_finished() below.

     concurrent_mark()->abort();

     // Make sure we'll choose a new allocation region afterwards.

     release_mutator_alloc_region();

     abandon_gc_alloc_regions();

     g1_rem_set()->cleanupHRRS();

     // We should call this after we retire any currently active alloc

     // regions so that all the ALLOC / RETIRE events are generated

     // before the start GC event.

     _hr_printer.start_gc(true /* full */, (size_t) total_collections());

     // We may have added regions to the current incremental collection

     // set between the last GC or pause and now. We need to clear the

     // incremental collection set and then start rebuilding it afresh

     // after this full GC.

     abandon_collection_set(g1_policy()->inc_cset_head());

     g1_policy()->clear_incremental_cset();

     g1_policy()->stop_incremental_cset_building();

     tear_down_region_sets(false /* free_list_only */);

     g1_policy()->set_gcs_are_young(true);

     // See the comments in g1CollectedHeap.hpp and

     // G1CollectedHeap::ref_processing_init() about

     // how reference processing currently works in G1.

     // Temporarily make discovery by the STW ref processor single threaded (non-MT).

     ReferenceProcessorMTDiscoveryMutator stw_rp_disc_ser(ref_processor_stw(), false);

     // Temporarily clear the STW ref processor's _is_alive_non_header field.

     ReferenceProcessorIsAliveMutator stw_rp_is_alive_null(ref_processor_stw(), NULL);

     ref_processor_stw()->enable_discovery(true /*verify_disabled*/, true /*verify_no_refs*/);

     ref_processor_stw()->setup_policy(do_clear_all_soft_refs);

     // Do collection work

     {

       HandleMark hm;  // Discard invalid handles created during gc

       G1MarkSweep::invoke_at_safepoint(ref_processor_stw(), do_clear_all_soft_refs);

     }

     assert(free_regions() == 0, "we should not have added any free regions");

     rebuild_region_sets(false /* free_list_only */);

     // Enqueue any discovered reference objects that have

     // not been removed from the discovered lists.

     ref_processor_stw()->enqueue_discovered_references();

     COMPILER2_PRESENT(DerivedPointerTable::update_pointers());

     MemoryService::track_memory_usage();

     if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       gclog_or_tty->print(" VerifyAfterGC:");

       prepare_for_verify();

       Universe::verify(/* silent      */ false,

                        /* option      */ VerifyOption_G1UsePrevMarking);

     }

     assert(!ref_processor_stw()->discovery_enabled(), "Postcondition");

     ref_processor_stw()->verify_no_references_recorded();

     // Note: since we've just done a full GC, concurrent

     // marking is no longer active. Therefore we need not

     // re-enable reference discovery for the CM ref processor.

     // That will be done at the start of the next marking cycle.

     assert(!ref_processor_cm()->discovery_enabled(), "Postcondition");

     ref_processor_cm()->verify_no_references_recorded();

     reset_gc_time_stamp();

     // Since everything potentially moved, we will clear all remembered

     // sets, and clear all cards.  Later we will rebuild remebered

     // sets. We will also reset the GC time stamps of the regions.

     PostMCRemSetClearClosure rs_clear(mr_bs());

     heap_region_iterate(&rs_clear);

     // Resize the heap if necessary.

     resize_if_necessary_after_full_collection(explicit_gc ? 0 : word_size);

     if (_hr_printer.is_active()) {

       // We should do this after we potentially resize the heap so

       // that all the COMMIT / UNCOMMIT events are generated before

       // the end GC event.

       PostCompactionPrinterClosure cl(hr_printer());

       heap_region_iterate(&cl);

       _hr_printer.end_gc(true /* full */, (size_t) total_collections());

     }

     if (_cg1r->use_cache()) {

       _cg1r->clear_and_record_card_counts();

       _cg1r->clear_hot_cache();

     }

     // Rebuild remembered sets of all regions.

     if (G1CollectedHeap::use_parallel_gc_threads()) {

       uint n_workers =

         AdaptiveSizePolicy::calc_active_workers(workers()->total_workers(),

                                        workers()->active_workers(),

                                        Threads::number_of_non_daemon_threads());

       assert(UseDynamicNumberOfGCThreads ||

              n_workers == workers()->total_workers(),

              "If not dynamic should be using all the  workers");

       workers()->set_active_workers(n_workers);

       // Set parallel threads in the heap (_n_par_threads) only

       // before a parallel phase and always reset it to 0 after

       // the phase so that the number of parallel threads does

       // no get carried forward to a serial phase where there

       // may be code that is "possibly_parallel".

       set_par_threads(n_workers);

       ParRebuildRSTask rebuild_rs_task(this);

       assert(check_heap_region_claim_values(

              HeapRegion::InitialClaimValue), "sanity check");

       assert(UseDynamicNumberOfGCThreads ||

              workers()->active_workers() == workers()->total_workers(),

         "Unless dynamic should use total workers");

       // Use the most recent number of  active workers

       assert(workers()->active_workers() > 0,

         "Active workers not properly set");

       set_par_threads(workers()->active_workers());

       workers()->run_task(&rebuild_rs_task);

       set_par_threads(0);

       assert(check_heap_region_claim_values(

              HeapRegion::RebuildRSClaimValue), "sanity check");

       reset_heap_region_claim_values();

     } else {

       RebuildRSOutOfRegionClosure rebuild_rs(this);

       heap_region_iterate(&rebuild_rs);

     }

     if (G1Log::fine()) {

       print_size_transition(gclog_or_tty, g1h_prev_used, used(), capacity());

     }

     if (true) { // FIXME

       // Ask the permanent generation to adjust size for full collections

       perm()->compute_new_size();

     }

     // Start a new incremental collection set for the next pause

     assert(g1_policy()->collection_set() == NULL, "must be");

     g1_policy()->start_incremental_cset_building();

     // Clear the _cset_fast_test bitmap in anticipation of adding

     // regions to the incremental collection set for the next

     // evacuation pause.

     clear_cset_fast_test();

     init_mutator_alloc_region();

     double end = os::elapsedTime();

     g1_policy()->record_full_collection_end();

 #ifdef TRACESPINNING

     ParallelTaskTerminator::print_termination_counts();

 #endif

     gc_epilogue(true);

     // Discard all rset updates

     JavaThread::dirty_card_queue_set().abandon_logs();

     assert(!G1DeferredRSUpdate

            || (G1DeferredRSUpdate && (dirty_card_queue_set().completed_buffers_num() == 0)), "Should not be any");

   }

   _young_list->reset_sampled_info();

   // At this point there should be no regions in the

   // entire heap tagged as young.

   assert( check_young_list_empty(true /* check_heap */),

     "young list should be empty at this point");

   // Update the number of full collections that have been completed.

   increment_full_collections_completed(false /* concurrent */);

   _hrs.verify_optional();

   verify_region_sets_optional();

   print_heap_after_gc();

   g1mm()->update_sizes();

   post_full_gc_dump();

   return true;

 }

 void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {

   // do_collection() will return whether it succeeded in performing

   // the GC. Currently, there is no facility on the

   // do_full_collection() API to notify the caller than the collection

   // did not succeed (e.g., because it was locked out by the GC

   // locker). So, right now, we'll ignore the return value.

   bool dummy = do_collection(true,                /* explicit_gc */

                              clear_all_soft_refs,

                              0                    /* word_size */);

 }

 // This code is mostly copied from TenuredGeneration.

 void

 G1CollectedHeap::

 resize_if_necessary_after_full_collection(size_t word_size) {

   assert(MinHeapFreeRatio <= MaxHeapFreeRatio, "sanity check");

   // Include the current allocation, if any, and bytes that will be

   // pre-allocated to support collections, as "used".

   const size_t used_after_gc = used();

   const size_t capacity_after_gc = capacity();

   const size_t free_after_gc = capacity_after_gc - used_after_gc;

   // This is enforced in arguments.cpp.

   assert(MinHeapFreeRatio <= MaxHeapFreeRatio,

          "otherwise the code below doesn't make sense");

   // We don't have floating point command-line arguments

   const double minimum_free_percentage = (double) MinHeapFreeRatio / 100.0;

   const double maximum_used_percentage = 1.0 - minimum_free_percentage;

   const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100.0;

   const double minimum_used_percentage = 1.0 - maximum_free_percentage;

   const size_t min_heap_size = collector_policy()->min_heap_byte_size();

   const size_t max_heap_size = collector_policy()->max_heap_byte_size();

   // We have to be careful here as these two calculations can overflow

   // 32-bit size_t's.

   double used_after_gc_d = (double) used_after_gc;

   double minimum_desired_capacity_d = used_after_gc_d / maximum_used_percentage;

   double maximum_desired_capacity_d = used_after_gc_d / minimum_used_percentage;

   // Let's make sure that they are both under the max heap size, which

   // by default will make them fit into a size_t.

   double desired_capacity_upper_bound = (double) max_heap_size;

   minimum_desired_capacity_d = MIN2(minimum_desired_capacity_d,

                                     desired_capacity_upper_bound);

   maximum_desired_capacity_d = MIN2(maximum_desired_capacity_d,

                                     desired_capacity_upper_bound);

   // We can now safely turn them into size_t's.

   size_t minimum_desired_capacity = (size_t) minimum_desired_capacity_d;

   size_t maximum_desired_capacity = (size_t) maximum_desired_capacity_d;

   // This assert only makes sense here, before we adjust them

   // with respect to the min and max heap size.

   assert(minimum_desired_capacity <= maximum_desired_capacity,

          err_msg("minimum_desired_capacity = "SIZE_FORMAT", "

                  "maximum_desired_capacity = "SIZE_FORMAT,

                  minimum_desired_capacity, maximum_desired_capacity));

   // Should not be greater than the heap max size. No need to adjust

   // it with respect to the heap min size as it's a lower bound (i.e.,

   // we'll try to make the capacity larger than it, not smaller).

   minimum_desired_capacity = MIN2(minimum_desired_capacity, max_heap_size);

   // Should not be less than the heap min size. No need to adjust it

   // with respect to the heap max size as it's an upper bound (i.e.,

   // we'll try to make the capacity smaller than it, not greater).

   maximum_desired_capacity =  MAX2(maximum_desired_capacity, min_heap_size);

   if (capacity_after_gc < minimum_desired_capacity) {

     // Don't expand unless it's significant

     size_t expand_bytes = minimum_desired_capacity - capacity_after_gc;

     ergo_verbose4(ErgoHeapSizing,

                   "attempt heap expansion",

                   ergo_format_reason("capacity lower than "

                                      "min desired capacity after Full GC")

                   ergo_format_byte("capacity")

                   ergo_format_byte("occupancy")

                   ergo_format_byte_perc("min desired capacity"),

                   capacity_after_gc, used_after_gc,

                   minimum_desired_capacity, (double) MinHeapFreeRatio);

     expand(expand_bytes);

     // No expansion, now see if we want to shrink

   } else if (capacity_after_gc > maximum_desired_capacity) {

     // Capacity too large, compute shrinking size

     size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity;

     ergo_verbose4(ErgoHeapSizing,

                   "attempt heap shrinking",

                   ergo_format_reason("capacity higher than "

                                      "max desired capacity after Full GC")

                   ergo_format_byte("capacity")

                   ergo_format_byte("occupancy")

                   ergo_format_byte_perc("max desired capacity"),

                   capacity_after_gc, used_after_gc,

                   maximum_desired_capacity, (double) MaxHeapFreeRatio);

     shrink(shrink_bytes);

   }

 }

 HeapWord*

 G1CollectedHeap::satisfy_failed_allocation(size_t word_size,

                                            bool* succeeded) {

   assert_at_safepoint(true /* should_be_vm_thread */);

   *succeeded = true;

   // Let's attempt the allocation first.

   HeapWord* result =

     attempt_allocation_at_safepoint(word_size,

                                  false /* expect_null_mutator_alloc_region */);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   // In a G1 heap, we're supposed to keep allocation from failing by

   // incremental pauses.  Therefore, at least for now, we'll favor

   // expansion over collection.  (This might change in the future if we can

   // do something smarter than full collection to satisfy a failed alloc.)

   result = expand_and_allocate(word_size);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   // Expansion didn't work, we'll try to do a Full GC.

   bool gc_succeeded = do_collection(false, /* explicit_gc */

                                     false, /* clear_all_soft_refs */

                                     word_size);

   if (!gc_succeeded) {

     *succeeded = false;

     return NULL;

   }

   // Retry the allocation

   result = attempt_allocation_at_safepoint(word_size,

                                   true /* expect_null_mutator_alloc_region */);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   // Then, try a Full GC that will collect all soft references.

   gc_succeeded = do_collection(false, /* explicit_gc */

                                true,  /* clear_all_soft_refs */

                                word_size);

   if (!gc_succeeded) {

     *succeeded = false;

     return NULL;

   }

   // Retry the allocation once more

   result = attempt_allocation_at_safepoint(word_size,

                                   true /* expect_null_mutator_alloc_region */);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   assert(!collector_policy()->should_clear_all_soft_refs(),

          "Flag should have been handled and cleared prior to this point");

   // What else?  We might try synchronous finalization later.  If the total

   // space available is large enough for the allocation, then a more

   // complete compaction phase than we've tried so far might be

   // appropriate.

   assert(*succeeded, "sanity");

   return NULL;

 }

 // Attempting to expand the heap sufficiently

 // to support an allocation of the given "word_size".  If

 // successful, perform the allocation and return the address of the

 // allocated block, or else "NULL".

 HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size) {

   assert_at_safepoint(true /* should_be_vm_thread */);

   verify_region_sets_optional();

   size_t expand_bytes = MAX2(word_size * HeapWordSize, MinHeapDeltaBytes);

   ergo_verbose1(ErgoHeapSizing,

                 "attempt heap expansion",

                 ergo_format_reason("allocation request failed")

                 ergo_format_byte("allocation request"),

                 word_size * HeapWordSize);

   if (expand(expand_bytes)) {

     _hrs.verify_optional();

     verify_region_sets_optional();

     return attempt_allocation_at_safepoint(word_size,

                                  false /* expect_null_mutator_alloc_region */);

   }

   return NULL;

 }

 void G1CollectedHeap::update_committed_space(HeapWord* old_end,

                                              HeapWord* new_end) {

   assert(old_end != new_end, "don't call this otherwise");

   assert((HeapWord*) _g1_storage.high() == new_end, "invariant");

   // Update the committed mem region.

   _g1_committed.set_end(new_end);

   // Tell the card table about the update.

   Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);

   // Tell the BOT about the update.

   _bot_shared->resize(_g1_committed.word_size());

 }

 bool G1CollectedHeap::expand(size_t expand_bytes) {

   size_t old_mem_size = _g1_storage.committed_size();

   size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes);

   aligned_expand_bytes = align_size_up(aligned_expand_bytes,

                                        HeapRegion::GrainBytes);

   ergo_verbose2(ErgoHeapSizing,

                 "expand the heap",

                 ergo_format_byte("requested expansion amount")

                 ergo_format_byte("attempted expansion amount"),

                 expand_bytes, aligned_expand_bytes);

   // First commit the memory.

   HeapWord* old_end = (HeapWord*) _g1_storage.high();

   bool successful = _g1_storage.expand_by(aligned_expand_bytes);

   if (successful) {

     // Then propagate this update to the necessary data structures.

     HeapWord* new_end = (HeapWord*) _g1_storage.high();

     update_committed_space(old_end, new_end);

     FreeRegionList expansion_list("Local Expansion List");

     MemRegion mr = _hrs.expand_by(old_end, new_end, &expansion_list);

     assert(mr.start() == old_end, "post-condition");

     // mr might be a smaller region than what was requested if

     // expand_by() was unable to allocate the HeapRegion instances

     assert(mr.end() <= new_end, "post-condition");

     size_t actual_expand_bytes = mr.byte_size();

     assert(actual_expand_bytes <= aligned_expand_bytes, "post-condition");

     assert(actual_expand_bytes == expansion_list.total_capacity_bytes(),

            "post-condition");

     if (actual_expand_bytes < aligned_expand_bytes) {

       // We could not expand _hrs to the desired size. In this case we

       // need to shrink the committed space accordingly.

       assert(mr.end() < new_end, "invariant");

       size_t diff_bytes = aligned_expand_bytes - actual_expand_bytes;

       // First uncommit the memory.

       _g1_storage.shrink_by(diff_bytes);

       // Then propagate this update to the necessary data structures.

       update_committed_space(new_end, mr.end());

     }

     _free_list.add_as_tail(&expansion_list);

     if (_hr_printer.is_active()) {

       HeapWord* curr = mr.start();

       while (curr < mr.end()) {

         HeapWord* curr_end = curr + HeapRegion::GrainWords;

         _hr_printer.commit(curr, curr_end);

         curr = curr_end;

       }

       assert(curr == mr.end(), "post-condition");

     }

     g1_policy()->record_new_heap_size(n_regions());

   } else {

     ergo_verbose0(ErgoHeapSizing,

                   "did not expand the heap",

                   ergo_format_reason("heap expansion operation failed"));

     // The expansion of the virtual storage space was unsuccessful.

     // Let's see if it was because we ran out of swap.

     if (G1ExitOnExpansionFailure &&

         _g1_storage.uncommitted_size() >= aligned_expand_bytes) {

       // We had head room...

       vm_exit_out_of_memory(aligned_expand_bytes, "G1 heap expansion");

     }

   }

   return successful;

 }

 void G1CollectedHeap::shrink_helper(size_t shrink_bytes) {

   size_t old_mem_size = _g1_storage.committed_size();

   size_t aligned_shrink_bytes =

     ReservedSpace::page_align_size_down(shrink_bytes);

   aligned_shrink_bytes = align_size_down(aligned_shrink_bytes,

                                          HeapRegion::GrainBytes);

   uint num_regions_deleted = 0;

   MemRegion mr = _hrs.shrink_by(aligned_shrink_bytes, &num_regions_deleted);

   HeapWord* old_end = (HeapWord*) _g1_storage.high();

   assert(mr.end() == old_end, "post-condition");

   ergo_verbose3(ErgoHeapSizing,

                 "shrink the heap",

                 ergo_format_byte("requested shrinking amount")

                 ergo_format_byte("aligned shrinking amount")

                 ergo_format_byte("attempted shrinking amount"),

                 shrink_bytes, aligned_shrink_bytes, mr.byte_size());

   if (mr.byte_size() > 0) {

     if (_hr_printer.is_active()) {

       HeapWord* curr = mr.end();

       while (curr > mr.start()) {

         HeapWord* curr_end = curr;

         curr -= HeapRegion::GrainWords;

         _hr_printer.uncommit(curr, curr_end);

       }

       assert(curr == mr.start(), "post-condition");

     }

     _g1_storage.shrink_by(mr.byte_size());

     HeapWord* new_end = (HeapWord*) _g1_storage.high();

     assert(mr.start() == new_end, "post-condition");

     _expansion_regions += num_regions_deleted;

     update_committed_space(old_end, new_end);

     HeapRegionRemSet::shrink_heap(n_regions());

     g1_policy()->record_new_heap_size(n_regions());

   } else {

     ergo_verbose0(ErgoHeapSizing,

                   "did not shrink the heap",

                   ergo_format_reason("heap shrinking operation failed"));

   }

 }

 void G1CollectedHeap::shrink(size_t shrink_bytes) {

   verify_region_sets_optional();

   // We should only reach here at the end of a Full GC which means we

   // should not not be holding to any GC alloc regions. The method

   // below will make sure of that and do any remaining clean up.

   abandon_gc_alloc_regions();

   // Instead of tearing down / rebuilding the free lists here, we

   // could instead use the remove_all_pending() method on free_list to

   // remove only the ones that we need to remove.

   tear_down_region_sets(true /* free_list_only */);

   shrink_helper(shrink_bytes);

   rebuild_region_sets(true /* free_list_only */);

   _hrs.verify_optional();

   verify_region_sets_optional();

 }

 // Public methods.

 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away

 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list

 #endif // _MSC_VER

 G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) :

   SharedHeap(policy_),

   _g1_policy(policy_),

   _dirty_card_queue_set(false),

   _into_cset_dirty_card_queue_set(false),

   _is_alive_closure_cm(this),

   _is_alive_closure_stw(this),

   _ref_processor_cm(NULL),

   _ref_processor_stw(NULL),

   _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)),

   _bot_shared(NULL),

   _objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL),

   _evac_failure_scan_stack(NULL) ,

   _mark_in_progress(false),

   _cg1r(NULL), _summary_bytes_used(0),

   _g1mm(NULL),

   _refine_cte_cl(NULL),

   _full_collection(false),

   _free_list("Master Free List"),

   _secondary_free_list("Secondary Free List"),

   _old_set("Old Set"),

   _humongous_set("Master Humongous Set"),

   _free_regions_coming(false),

   _young_list(new YoungList(this)),

   _gc_time_stamp(0),

   _retained_old_gc_alloc_region(NULL),

   _expand_heap_after_alloc_failure(true),

   _surviving_young_words(NULL),

   _full_collections_completed(0),

   _in_cset_fast_test(NULL),

   _in_cset_fast_test_base(NULL),

   _dirty_cards_region_list(NULL),

   _worker_cset_start_region(NULL),

   _worker_cset_start_region_time_stamp(NULL) {

   _g1h = this; // To catch bugs.

   if (_process_strong_tasks == NULL || !_process_strong_tasks->valid()) {

     vm_exit_during_initialization("Failed necessary allocation.");

   }

   _humongous_object_threshold_in_words = HeapRegion::GrainWords / 2;

   int n_queues = MAX2((int)ParallelGCThreads, 1);

   _task_queues = new RefToScanQueueSet(n_queues);

   int n_rem_sets = HeapRegionRemSet::num_par_rem_sets();

   assert(n_rem_sets > 0, "Invariant.");

   HeapRegionRemSetIterator** iter_arr =

     NEW_C_HEAP_ARRAY(HeapRegionRemSetIterator*, n_queues);

   for (int i = 0; i < n_queues; i++) {

     iter_arr[i] = new HeapRegionRemSetIterator();

   }

   _rem_set_iterator = iter_arr;

   _worker_cset_start_region = NEW_C_HEAP_ARRAY(HeapRegion*, n_queues);

   _worker_cset_start_region_time_stamp = NEW_C_HEAP_ARRAY(unsigned int, n_queues);

   for (int i = 0; i < n_queues; i++) {

     RefToScanQueue* q = new RefToScanQueue();

     q->initialize();

     _task_queues->register_queue(i, q);

   }

   clear_cset_start_regions();

   guarantee(_task_queues != NULL, "task_queues allocation failure.");

 }

 jint G1CollectedHeap::initialize() {

   CollectedHeap::pre_initialize();

   os::enable_vtime();

   G1Log::init();

   // Necessary to satisfy locking discipline assertions.

   MutexLocker x(Heap_lock);

   // We have to initialize the printer before committing the heap, as

   // it will be used then.

   _hr_printer.set_active(G1PrintHeapRegions);

   // While there are no constraints in the GC code that HeapWordSize

   // be any particular value, there are multiple other areas in the

   // system which believe this to be true (e.g. oop->object_size in some

   // cases incorrectly returns the size in wordSize units rather than

   // HeapWordSize).

   guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize");

   size_t init_byte_size = collector_policy()->initial_heap_byte_size();

   size_t max_byte_size = collector_policy()->max_heap_byte_size();

   // Ensure that the sizes are properly aligned.

   Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap");

   Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap");

   _cg1r = new ConcurrentG1Refine();

   // Reserve the maximum.

   PermanentGenerationSpec* pgs = collector_policy()->permanent_generation();

   // Includes the perm-gen.

   // When compressed oops are enabled, the preferred heap base

   // is calculated by subtracting the requested size from the

   // 32Gb boundary and using the result as the base address for

   // heap reservation. If the requested size is not aligned to

   // HeapRegion::GrainBytes (i.e. the alignment that is passed

   // into the ReservedHeapSpace constructor) then the actual

   // base of the reserved heap may end up differing from the

   // address that was requested (i.e. the preferred heap base).

   // If this happens then we could end up using a non-optimal

   // compressed oops mode.

   // Since max_byte_size is aligned to the size of a heap region (checked

   // above), we also need to align the perm gen size as it might not be.

   const size_t total_reserved = max_byte_size +

                                 align_size_up(pgs->max_size(), HeapRegion::GrainBytes);

   Universe::check_alignment(total_reserved, HeapRegion::GrainBytes, "g1 heap and perm");

   char* addr = Universe::preferred_heap_base(total_reserved, Universe::UnscaledNarrowOop);

   ReservedHeapSpace heap_rs(total_reserved, HeapRegion::GrainBytes,

                             UseLargePages, addr);

   if (UseCompressedOops) {

     if (addr != NULL && !heap_rs.is_reserved()) {

       // Failed to reserve at specified address - the requested memory

       // region is taken already, for example, by 'java' launcher.

       // Try again to reserver heap higher.

       addr = Universe::preferred_heap_base(total_reserved, Universe::ZeroBasedNarrowOop);

       ReservedHeapSpace heap_rs0(total_reserved, HeapRegion::GrainBytes,

                                  UseLargePages, addr);

       if (addr != NULL && !heap_rs0.is_reserved()) {

         // Failed to reserve at specified address again - give up.

         addr = Universe::preferred_heap_base(total_reserved, Universe::HeapBasedNarrowOop);

         assert(addr == NULL, "");

         ReservedHeapSpace heap_rs1(total_reserved, HeapRegion::GrainBytes,

                                    UseLargePages, addr);

         heap_rs = heap_rs1;

       } else {

         heap_rs = heap_rs0;

       }

     }

   }

   if (!heap_rs.is_reserved()) {

     vm_exit_during_initialization("Could not reserve enough space for object heap");

     return JNI_ENOMEM;

   }

   // It is important to do this in a way such that concurrent readers can't

   // temporarily think somethings in the heap.  (I've actually seen this

   // happen in asserts: DLD.)

   _reserved.set_word_size(0);

   _reserved.set_start((HeapWord*)heap_rs.base());

   _reserved.set_end((HeapWord*)(heap_rs.base() + heap_rs.size()));

   _expansion_regions = (uint) (max_byte_size / HeapRegion::GrainBytes);

   // Create the gen rem set (and barrier set) for the entire reserved region.

   _rem_set = collector_policy()->create_rem_set(_reserved, 2);

   set_barrier_set(rem_set()->bs());

   if (barrier_set()->is_a(BarrierSet::ModRef)) {

     _mr_bs = (ModRefBarrierSet*)_barrier_set;

   } else {

     vm_exit_during_initialization("G1 requires a mod ref bs.");

     return JNI_ENOMEM;

   }

   // Also create a G1 rem set.

   if (mr_bs()->is_a(BarrierSet::CardTableModRef)) {

     _g1_rem_set = new G1RemSet(this, (CardTableModRefBS*)mr_bs());

   } else {

     vm_exit_during_initialization("G1 requires a cardtable mod ref bs.");

     return JNI_ENOMEM;

   }

   // Carve out the G1 part of the heap.

   ReservedSpace g1_rs   = heap_rs.first_part(max_byte_size);

   _g1_reserved = MemRegion((HeapWord*)g1_rs.base(),

                            g1_rs.size()/HeapWordSize);

   ReservedSpace perm_gen_rs = heap_rs.last_part(max_byte_size);

   _perm_gen = pgs->init(perm_gen_rs, pgs->init_size(), rem_set());

   _g1_storage.initialize(g1_rs, 0);

   _g1_committed = MemRegion((HeapWord*)_g1_storage.low(), (size_t) 0);

   _hrs.initialize((HeapWord*) _g1_reserved.start(),

                   (HeapWord*) _g1_reserved.end(),

                   _expansion_regions);

   // 6843694 - ensure that the maximum region index can fit

   // in the remembered set structures.

   const uint max_region_idx = (1U << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1;

   guarantee((max_regions() - 1) <= max_region_idx, "too many regions");

   size_t max_cards_per_region = ((size_t)1 << (sizeof(CardIdx_t)*BitsPerByte-1)) - 1;

   guarantee(HeapRegion::CardsPerRegion > 0, "make sure it's initialized");

   guarantee(HeapRegion::CardsPerRegion < max_cards_per_region,

             "too many cards per region");

   HeapRegionSet::set_unrealistically_long_length(max_regions() + 1);

   _bot_shared = new G1BlockOffsetSharedArray(_reserved,

                                              heap_word_size(init_byte_size));

   _g1h = this;

    _in_cset_fast_test_length = max_regions();

    _in_cset_fast_test_base =

                    NEW_C_HEAP_ARRAY(bool, (size_t) _in_cset_fast_test_length);

    // We're biasing _in_cset_fast_test to avoid subtracting the

    // beginning of the heap every time we want to index; basically

    // it's the same with what we do with the card table.

    _in_cset_fast_test = _in_cset_fast_test_base -

                ((uintx) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);

    // Clear the _cset_fast_test bitmap in anticipation of adding

    // regions to the incremental collection set for the first

    // evacuation pause.

    clear_cset_fast_test();

   // Create the ConcurrentMark data structure and thread.

   // (Must do this late, so that "max_regions" is defined.)

   _cm       = new ConcurrentMark(heap_rs, max_regions());

   _cmThread = _cm->cmThread();

   // Initialize the from_card cache structure of HeapRegionRemSet.

   HeapRegionRemSet::init_heap(max_regions());

   // Now expand into the initial heap size.

   if (!expand(init_byte_size)) {

     vm_exit_during_initialization("Failed to allocate initial heap.");

     return JNI_ENOMEM;

   }

   // Perform any initialization actions delegated to the policy.

   g1_policy()->init();

   _refine_cte_cl =

     new RefineCardTableEntryClosure(ConcurrentG1RefineThread::sts(),

                                     g1_rem_set(),

                                     concurrent_g1_refine());

   JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

   JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,

                                                SATB_Q_FL_lock,

                                                G1SATBProcessCompletedThreshold,

                                                Shared_SATB_Q_lock);

   JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,

                                                 DirtyCardQ_FL_lock,

                                                 concurrent_g1_refine()->yellow_zone(),

                                                 concurrent_g1_refine()->red_zone(),

                                                 Shared_DirtyCardQ_lock);

   if (G1DeferredRSUpdate) {

     dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,

                                       DirtyCardQ_FL_lock,

                                       -1, // never trigger processing

                                       -1, // no limit on length

                                       Shared_DirtyCardQ_lock,

                                       &JavaThread::dirty_card_queue_set());

   }

   // Initialize the card queue set used to hold cards containing

   // references into the collection set.

   _into_cset_dirty_card_queue_set.initialize(DirtyCardQ_CBL_mon,

                                              DirtyCardQ_FL_lock,

                                              -1, // never trigger processing

                                              -1, // no limit on length

                                              Shared_DirtyCardQ_lock,

                                              &JavaThread::dirty_card_queue_set());

   // In case we're keeping closure specialization stats, initialize those

   // counts and that mechanism.

   SpecializationStats::clear();

   // Do later initialization work for concurrent refinement.

   _cg1r->init();

   // Here we allocate the dummy full region that is required by the

   // G1AllocRegion class. If we don't pass an address in the reserved

   // space here, lots of asserts fire.

   HeapRegion* dummy_region = new_heap_region(0 /* index of bottom region */,

                                              _g1_reserved.start());

   // We'll re-use the same region whether the alloc region will

   // require BOT updates or not and, if it doesn't, then a non-young

   // region will complain that it cannot support allocations without

   // BOT updates. So we'll tag the dummy region as young to avoid that.

   dummy_region->set_young();

   // Make sure it's full.

   dummy_region->set_top(dummy_region->end());

   G1AllocRegion::setup(this, dummy_region);

   init_mutator_alloc_region();

   // Do create of the monitoring and management support so that

   // values in the heap have been properly initialized.

   _g1mm = new G1MonitoringSupport(this);

   return JNI_OK;

 }

 void G1CollectedHeap::ref_processing_init() {

   // Reference processing in G1 currently works as follows:

   //

   // * There are two reference processor instances. One is

   //   used to record and process discovered references

   //   during concurrent marking; the other is used to

   //   record and process references during STW pauses

   //   (both full and incremental).

   // * Both ref processors need to 'span' the entire heap as

   //   the regions in the collection set may be dotted around.

   //

   // * For the concurrent marking ref processor:

   //   * Reference discovery is enabled at initial marking.

   //   * Reference discovery is disabled and the discovered

   //     references processed etc during remarking.

   //   * Reference discovery is MT (see below).

   //   * Reference discovery requires a barrier (see below).

   //   * Reference processing may or may not be MT

   //     (depending on the value of ParallelRefProcEnabled

   //     and ParallelGCThreads).

   //   * A full GC disables reference discovery by the CM

   //     ref processor and abandons any entries on it's

   //     discovered lists.

   //

   // * For the STW processor:

   //   * Non MT discovery is enabled at the start of a full GC.

   //   * Processing and enqueueing during a full GC is non-MT.

   //   * During a full GC, references are processed after marking.

   //

   //   * Discovery (may or may not be MT) is enabled at the start

   //     of an incremental evacuation pause.

   //   * References are processed near the end of a STW evacuation pause.

   //   * For both types of GC:

   //     * Discovery is atomic - i.e. not concurrent.

   //     * Reference discovery will not need a barrier.

   SharedHeap::ref_processing_init();

   MemRegion mr = reserved_region();

   // Concurrent Mark ref processor

   _ref_processor_cm =

     new ReferenceProcessor(mr,    // span

                            ParallelRefProcEnabled && (ParallelGCThreads > 1),

                                 // mt processing

                            (int) ParallelGCThreads,

                                 // degree of mt processing

                            (ParallelGCThreads > 1) || (ConcGCThreads > 1),

                                 // mt discovery

                            (int) MAX2(ParallelGCThreads, ConcGCThreads),

                                 // degree of mt discovery

                            false,

                                 // Reference discovery is not atomic

                            &_is_alive_closure_cm,

                                 // is alive closure

                                 // (for efficiency/performance)

                            true);

                                 // Setting next fields of discovered

                                 // lists requires a barrier.

   // STW ref processor

   _ref_processor_stw =

     new ReferenceProcessor(mr,    // span

                            ParallelRefProcEnabled && (ParallelGCThreads > 1),

                                 // mt processing

                            MAX2((int)ParallelGCThreads, 1),

                                 // degree of mt processing

                            (ParallelGCThreads > 1),

                                 // mt discovery

                            MAX2((int)ParallelGCThreads, 1),

                                 // degree of mt discovery

                            true,

                                 // Reference discovery is atomic

                            &_is_alive_closure_stw,

                                 // is alive closure

                                 // (for efficiency/performance)

                            false);

                                 // Setting next fields of discovered

                                 // lists requires a barrier.

 }

 size_t G1CollectedHeap::capacity() const {

   return _g1_committed.byte_size();

 }

 void G1CollectedHeap::iterate_dirty_card_closure(CardTableEntryClosure* cl,

                                                  DirtyCardQueue* into_cset_dcq,

                                                  bool concurrent,

                                                  int worker_i) {

   // Clean cards in the hot card cache

   concurrent_g1_refine()->clean_up_cache(worker_i, g1_rem_set(), into_cset_dcq);

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   int n_completed_buffers = 0;

   while (dcqs.apply_closure_to_completed_buffer(cl, worker_i, 0, true)) {

     n_completed_buffers++;

   }

   g1_policy()->record_update_rs_processed_buffers(worker_i,

                                                   (double) n_completed_buffers);

   dcqs.clear_n_completed_buffers();

   assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!");

 }

 // Computes the sum of the storage used by the various regions.

 size_t G1CollectedHeap::used() const {

   assert(Heap_lock->owner() != NULL,

          "Should be owned on this thread's behalf.");

   size_t result = _summary_bytes_used;

   // Read only once in case it is set to NULL concurrently

   HeapRegion* hr = _mutator_alloc_region.get();

   if (hr != NULL)

     result += hr->used();

   return result;

 }

 size_t G1CollectedHeap::used_unlocked() const {

   size_t result = _summary_bytes_used;

   return result;

 }

 class SumUsedClosure: public HeapRegionClosure {

   size_t _used;

 public:

   SumUsedClosure() : _used(0) {}

   bool doHeapRegion(HeapRegion* r) {

     if (!r->continuesHumongous()) {

       _used += r->used();

     }

     return false;

   }

   size_t result() { return _used; }

 };

 size_t G1CollectedHeap::recalculate_used() const {

   SumUsedClosure blk;

   heap_region_iterate(&blk);

   return blk.result();

 }

 size_t G1CollectedHeap::unsafe_max_alloc() {

   if (free_regions() > 0) return HeapRegion::GrainBytes;

   // otherwise, is there space in the current allocation region?

   // We need to store the current allocation region in a local variable

   // here. The problem is that this method doesn't take any locks and

   // there may be other threads which overwrite the current allocation

   // region field. attempt_allocation(), for example, sets it to NULL

   // and this can happen *after* the NULL check here but before the call

   // to free(), resulting in a SIGSEGV. Note that this doesn't appear

   // to be a problem in the optimized build, since the two loads of the

   // current allocation region field are optimized away.

   HeapRegion* hr = _mutator_alloc_region.get();

   if (hr == NULL) {

     return 0;

   }

   return hr->free();

 }

 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {

   switch (cause) {

     case GCCause::_gc_locker:               return GCLockerInvokesConcurrent;

     case GCCause::_java_lang_system_gc:     return ExplicitGCInvokesConcurrent;

     case GCCause::_g1_humongous_allocation: return true;

     default:                                return false;

   }

 }

 #ifndef PRODUCT

 void G1CollectedHeap::allocate_dummy_regions() {

   // Let's fill up most of the region

   size_t word_size = HeapRegion::GrainWords - 1024;

   // And as a result the region we'll allocate will be humongous.

   guarantee(isHumongous(word_size), "sanity");

   for (uintx i = 0; i < G1DummyRegionsPerGC; ++i) {

     // Let's use the existing mechanism for the allocation

     HeapWord* dummy_obj = humongous_obj_allocate(word_size);

     if (dummy_obj != NULL) {

       MemRegion mr(dummy_obj, word_size);

       CollectedHeap::fill_with_object(mr);

     } else {

       // If we can't allocate once, we probably cannot allocate

       // again. Let's get out of the loop.

       break;

     }

   }

 }

 #endif // !PRODUCT

 void G1CollectedHeap::increment_full_collections_completed(bool concurrent) {

   MonitorLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag);

   // We assume that if concurrent == true, then the caller is a

   // concurrent thread that was joined the Suspendible Thread

   // Set. If there's ever a cheap way to check this, we should add an

   // assert here.

   // We have already incremented _total_full_collections at the start

   // of the GC, so total_full_collections() represents how many full

   // collections have been started.

   unsigned int full_collections_started = total_full_collections();

   // Given that this method is called at the end of a Full GC or of a

   // concurrent cycle, and those can be nested (i.e., a Full GC can

   // interrupt a concurrent cycle), the number of full collections

   // completed should be either one (in the case where there was no

   // nesting) or two (when a Full GC interrupted a concurrent cycle)

   // behind the number of full collections started.

   // This is the case for the inner caller, i.e. a Full GC.

   assert(concurrent ||

          (full_collections_started == _full_collections_completed + 1) ||

          (full_collections_started == _full_collections_completed + 2),

          err_msg("for inner caller (Full GC): full_collections_started = %u "

                  "is inconsistent with _full_collections_completed = %u",

                  full_collections_started, _full_collections_completed));

   // This is the case for the outer caller, i.e. the concurrent cycle.

   assert(!concurrent ||

          (full_collections_started == _full_collections_completed + 1),

          err_msg("for outer caller (concurrent cycle): "

                  "full_collections_started = %u "

                  "is inconsistent with _full_collections_completed = %u",

                  full_collections_started, _full_collections_completed));

   _full_collections_completed += 1;

   // We need to clear the "in_progress" flag in the CM thread before

   // we wake up any waiters (especially when ExplicitInvokesConcurrent

   // is set) so that if a waiter requests another System.gc() it doesn't

   // incorrectly see that a marking cyle is still in progress.

   if (concurrent) {

     _cmThread->clear_in_progress();

   }

   // This notify_all() will ensure that a thread that called

   // System.gc() with (with ExplicitGCInvokesConcurrent set or not)

   // and it's waiting for a full GC to finish will be woken up. It is

   // waiting in VM_G1IncCollectionPause::doit_epilogue().

   FullGCCount_lock->notify_all();

 }

 void G1CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) {

   assert_at_safepoint(true /* should_be_vm_thread */);

   GCCauseSetter gcs(this, cause);

   switch (cause) {

     case GCCause::_heap_inspection:

     case GCCause::_heap_dump: {

       HandleMark hm;

       do_full_collection(false);         // don't clear all soft refs

       break;

     }

     default: // XXX FIX ME

       ShouldNotReachHere(); // Unexpected use of this function

   }

 }

 void G1CollectedHeap::collect(GCCause::Cause cause) {

   assert_heap_not_locked();

   unsigned int gc_count_before;

   unsigned int full_gc_count_before;

   bool retry_gc;

   do {

     retry_gc = false;

     {

       MutexLocker ml(Heap_lock);

       // Read the GC count while holding the Heap_lock

       gc_count_before = total_collections();

       full_gc_count_before = total_full_collections();

     }

     if (should_do_concurrent_full_gc(cause)) {

       // Schedule an initial-mark evacuation pause that will start a

       // concurrent cycle. We're setting word_size to 0 which means that

       // we are not requesting a post-GC allocation.

       VM_G1IncCollectionPause op(gc_count_before,

                                  0,     /* word_size */

                                  true,  /* should_initiate_conc_mark */

                                  g1_policy()->max_pause_time_ms(),

                                  cause);

       VMThread::execute(&op);

       if (!op.pause_succeeded()) {

         if (full_gc_count_before == total_full_collections()) {

           retry_gc = op.should_retry_gc();

         } else {

           // A Full GC happened while we were trying to schedule the

           // initial-mark GC. No point in starting a new cycle given

           // that the whole heap was collected anyway.

         }

         if (retry_gc) {

           if (GC_locker::is_active_and_needs_gc()) {

             GC_locker::stall_until_clear();

           }

         }

       }

     } else {

       if (cause == GCCause::_gc_locker

           DEBUG_ONLY(|| cause == GCCause::_scavenge_alot)) {

         // Schedule a standard evacuation pause. We're setting word_size

         // to 0 which means that we are not requesting a post-GC allocation.

         VM_G1IncCollectionPause op(gc_count_before,

                                    0,     /* word_size */

                                    false, /* should_initiate_conc_mark */

                                    g1_policy()->max_pause_time_ms(),

                                    cause);

         VMThread::execute(&op);

       } else {

         // Schedule a Full GC.

         VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);

         VMThread::execute(&op);

       }

     }

   } while (retry_gc);

 }

 bool G1CollectedHeap::is_in(const void* p) const {

   if (_g1_committed.contains(p)) {

     // Given that we know that p is in the committed space,

     // heap_region_containing_raw() should successfully

     // return the containing region.

     HeapRegion* hr = heap_region_containing_raw(p);

     return hr->is_in(p);

   } else {

     return _perm_gen->as_gen()->is_in(p);

   }

 }

 // Iteration functions.

 // Iterates an OopClosure over all ref-containing fields of objects

 // within a HeapRegion.

 class IterateOopClosureRegionClosure: public HeapRegionClosure {

   MemRegion _mr;

   OopClosure* _cl;

 public:

   IterateOopClosureRegionClosure(MemRegion mr, OopClosure* cl)

     : _mr(mr), _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     if (! r->continuesHumongous()) {

       r->oop_iterate(_cl);

     }

     return false;

   }

 };

 void G1CollectedHeap::oop_iterate(OopClosure* cl, bool do_perm) {

   IterateOopClosureRegionClosure blk(_g1_committed, cl);

   heap_region_iterate(&blk);

   if (do_perm) {

     perm_gen()->oop_iterate(cl);

   }

 }

 void G1CollectedHeap::oop_iterate(MemRegion mr, OopClosure* cl, bool do_perm) {

   IterateOopClosureRegionClosure blk(mr, cl);

   heap_region_iterate(&blk);

   if (do_perm) {

     perm_gen()->oop_iterate(cl);

   }

 }

 // Iterates an ObjectClosure over all objects within a HeapRegion.

 class IterateObjectClosureRegionClosure: public HeapRegionClosure {

   ObjectClosure* _cl;

 public:

   IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     if (! r->continuesHumongous()) {

       r->object_iterate(_cl);

     }

     return false;

   }

 };

 void G1CollectedHeap::object_iterate(ObjectClosure* cl, bool do_perm) {

   IterateObjectClosureRegionClosure blk(cl);

   heap_region_iterate(&blk);

   if (do_perm) {

     perm_gen()->object_iterate(cl);

   }

 }

 void G1CollectedHeap::object_iterate_since_last_GC(ObjectClosure* cl) {

   // FIXME: is this right?

   guarantee(false, "object_iterate_since_last_GC not supported by G1 heap");

 }

 // Calls a SpaceClosure on a HeapRegion.

 class SpaceClosureRegionClosure: public HeapRegionClosure {

   SpaceClosure* _cl;

 public:

   SpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     _cl->do_space(r);

     return false;

   }

 };

 void G1CollectedHeap::space_iterate(SpaceClosure* cl) {

   SpaceClosureRegionClosure blk(cl);

   heap_region_iterate(&blk);

 }

 void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const {

   _hrs.iterate(cl);

 }

 void G1CollectedHeap::heap_region_iterate_from(HeapRegion* r,

                                                HeapRegionClosure* cl) const {

   _hrs.iterate_from(r, cl);

 }

 void

 G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl,

                                                  uint worker,

                                                  uint no_of_par_workers,

                                                  jint claim_value) {

   const uint regions = n_regions();

   const uint max_workers = (G1CollectedHeap::use_parallel_gc_threads() ?

                              no_of_par_workers :

                              1);

   assert(UseDynamicNumberOfGCThreads ||

          no_of_par_workers == workers()->total_workers(),

          "Non dynamic should use fixed number of workers");

   // try to spread out the starting points of the workers

   const uint start_index = regions / max_workers * worker;

   // each worker will actually look at all regions

   for (uint count = 0; count < regions; ++count) {

     const uint index = (start_index + count) % regions;

     assert(0 <= index && index < regions, "sanity");

     HeapRegion* r = region_at(index);

     // we'll ignore "continues humongous" regions (we'll process them

     // when we come across their corresponding "start humongous"

     // region) and regions already claimed

     if (r->claim_value() == claim_value || r->continuesHumongous()) {

       continue;

     }

     // OK, try to claim it

     if (r->claimHeapRegion(claim_value)) {

       // success!

       assert(!r->continuesHumongous(), "sanity");

       if (r->startsHumongous()) {

         // If the region is "starts humongous" we'll iterate over its

         // "continues humongous" first; in fact we'll do them

         // first. The order is important. In on case, calling the

         // closure on the "starts humongous" region might de-allocate

         // and clear all its "continues humongous" regions and, as a

         // result, we might end up processing them twice. So, we'll do

         // them first (notice: most closures will ignore them anyway) and

         // then we'll do the "starts humongous" region.

         for (uint ch_index = index + 1; ch_index < regions; ++ch_index) {

           HeapRegion* chr = region_at(ch_index);

           // if the region has already been claimed or it's not

           // "continues humongous" we're done

           if (chr->claim_value() == claim_value ||

               !chr->continuesHumongous()) {

             break;

           }

           // Noone should have claimed it directly. We can given

           // that we claimed its "starts humongous" region.

           assert(chr->claim_value() != claim_value, "sanity");

           assert(chr->humongous_start_region() == r, "sanity");

           if (chr->claimHeapRegion(claim_value)) {

             // we should always be able to claim it; noone else should

             // be trying to claim this region

             bool res2 = cl->doHeapRegion(chr);

             assert(!res2, "Should not abort");

             // Right now, this holds (i.e., no closure that actually

             // does something with "continues humongous" regions

             // clears them). We might have to weaken it in the future,

             // but let's leave these two asserts here for extra safety.

             assert(chr->continuesHumongous(), "should still be the case");

             assert(chr->humongous_start_region() == r, "sanity");

           } else {

             guarantee(false, "we should not reach here");

           }

         }