Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "precompiled.hpp"

 #include "code/icBuffer.hpp"

 #include "gc_implementation/g1/bufferingOopClosure.hpp"

 #include "gc_implementation/g1/concurrentG1Refine.hpp"

 #include "gc_implementation/g1/concurrentG1RefineThread.hpp"

 #include "gc_implementation/g1/concurrentMarkThread.inline.hpp"

 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

 #include "gc_implementation/g1/g1CollectorPolicy.hpp"

 #include "gc_implementation/g1/g1MarkSweep.hpp"

 #include "gc_implementation/g1/g1OopClosures.inline.hpp"

 #include "gc_implementation/g1/g1RemSet.inline.hpp"

 #include "gc_implementation/g1/heapRegionRemSet.hpp"

 #include "gc_implementation/g1/heapRegionSeq.inline.hpp"

 #include "gc_implementation/g1/vm_operations_g1.hpp"

 #include "gc_implementation/shared/isGCActiveMark.hpp"

 #include "memory/gcLocker.inline.hpp"

 #include "memory/genOopClosures.inline.hpp"

 #include "memory/generationSpec.hpp"

 #include "oops/oop.inline.hpp"

 #include "oops/oop.pcgc.inline.hpp"

 #include "runtime/aprofiler.hpp"

 #include "runtime/vmThread.hpp"

 size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0;

 // turn it on so that the contents of the young list (scan-only /

 // to-be-collected) are printed at "strategic" points before / during

 // / after the collection --- this is useful for debugging

 #define YOUNG_LIST_VERBOSE 0

 // CURRENT STATUS

 // This file is under construction.  Search for "FIXME".

 // INVARIANTS/NOTES

 //

 // All allocation activity covered by the G1CollectedHeap interface is

 // serialized by acquiring the HeapLock.  This happens in mem_allocate

 // and allocate_new_tlab, which are the "entry" points to the

 // allocation code from the rest of the JVM.  (Note that this does not

 // apply to TLAB allocation, which is not part of this interface: it

 // is done by clients of this interface.)

 // Local to this file.

 class RefineCardTableEntryClosure: public CardTableEntryClosure {

   SuspendibleThreadSet* _sts;

   G1RemSet* _g1rs;

   ConcurrentG1Refine* _cg1r;

   bool _concurrent;

 public:

   RefineCardTableEntryClosure(SuspendibleThreadSet* sts,

                               G1RemSet* g1rs,

                               ConcurrentG1Refine* cg1r) :

     _sts(sts), _g1rs(g1rs), _cg1r(cg1r), _concurrent(true)

   {}

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     bool oops_into_cset = _g1rs->concurrentRefineOneCard(card_ptr, worker_i, false);

     // This path is executed by the concurrent refine or mutator threads,

     // concurrently, and so we do not care if card_ptr contains references

     // that point into the collection set.

     assert(!oops_into_cset, "should be");

     if (_concurrent && _sts->should_yield()) {

       // Caller will actually yield.

       return false;

     }

     // Otherwise, we finished successfully; return true.

     return true;

   }

   void set_concurrent(bool b) { _concurrent = b; }

 };

 class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure {

   int _calls;

   G1CollectedHeap* _g1h;

   CardTableModRefBS* _ctbs;

   int _histo[256];

 public:

   ClearLoggedCardTableEntryClosure() :

     _calls(0)

   {

     _g1h = G1CollectedHeap::heap();

     _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

     for (int i = 0; i < 256; i++) _histo[i] = 0;

   }

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

       _calls++;

       unsigned char* ujb = (unsigned char*)card_ptr;

       int ind = (int)(*ujb);

       _histo[ind]++;

       *card_ptr = -1;

     }

     return true;

   }

   int calls() { return _calls; }

   void print_histo() {

     gclog_or_tty->print_cr("Card table value histogram:");

     for (int i = 0; i < 256; i++) {

       if (_histo[i] != 0) {

         gclog_or_tty->print_cr("  %d: %d", i, _histo[i]);

       }

     }

   }

 };

 class RedirtyLoggedCardTableEntryClosure: public CardTableEntryClosure {

   int _calls;

   G1CollectedHeap* _g1h;

   CardTableModRefBS* _ctbs;

 public:

   RedirtyLoggedCardTableEntryClosure() :

     _calls(0)

   {

     _g1h = G1CollectedHeap::heap();

     _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

   }

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

       _calls++;

       *card_ptr = 0;

     }

     return true;

   }

   int calls() { return _calls; }

 };

 class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure {

 public:

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     *card_ptr = CardTableModRefBS::dirty_card_val();

     return true;

   }

 };

 YoungList::YoungList(G1CollectedHeap* g1h)

   : _g1h(g1h), _head(NULL),

     _length(0),

     _last_sampled_rs_lengths(0),

     _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0)

 {

   guarantee( check_list_empty(false), "just making sure..." );

 }

 void YoungList::push_region(HeapRegion *hr) {

   assert(!hr->is_young(), "should not already be young");

   assert(hr->get_next_young_region() == NULL, "cause it should!");

   hr->set_next_young_region(_head);

   _head = hr;

   hr->set_young();

   double yg_surv_rate = _g1h->g1_policy()->predict_yg_surv_rate((int)_length);

   ++_length;

 }

 void YoungList::add_survivor_region(HeapRegion* hr) {

   assert(hr->is_survivor(), "should be flagged as survivor region");

   assert(hr->get_next_young_region() == NULL, "cause it should!");

   hr->set_next_young_region(_survivor_head);

   if (_survivor_head == NULL) {

     _survivor_tail = hr;

   }

   _survivor_head = hr;

   ++_survivor_length;

 }

 void YoungList::empty_list(HeapRegion* list) {

   while (list != NULL) {

     HeapRegion* next = list->get_next_young_region();

     list->set_next_young_region(NULL);

     list->uninstall_surv_rate_group();

     list->set_not_young();

     list = next;

   }

 }

 void YoungList::empty_list() {

   assert(check_list_well_formed(), "young list should be well formed");

   empty_list(_head);

   _head = NULL;

   _length = 0;

   empty_list(_survivor_head);

   _survivor_head = NULL;

   _survivor_tail = NULL;

   _survivor_length = 0;

   _last_sampled_rs_lengths = 0;

   assert(check_list_empty(false), "just making sure...");

 }

 bool YoungList::check_list_well_formed() {

   bool ret = true;

   size_t length = 0;

   HeapRegion* curr = _head;

   HeapRegion* last = NULL;

   while (curr != NULL) {

     if (!curr->is_young()) {

       gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" "

                              "incorrectly tagged (y: %d, surv: %d)",

                              curr->bottom(), curr->end(),

                              curr->is_young(), curr->is_survivor());

       ret = false;

     }

     ++length;

     last = curr;

     curr = curr->get_next_young_region();

   }

   ret = ret && (length == _length);

   if (!ret) {

     gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!");

     gclog_or_tty->print_cr("###   list has %d entries, _length is %d",

                            length, _length);

   }

   return ret;

 }

 bool YoungList::check_list_empty(bool check_sample) {

   bool ret = true;

   if (_length != 0) {

     gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %d",

                   _length);

     ret = false;

   }

   if (check_sample && _last_sampled_rs_lengths != 0) {

     gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths");

     ret = false;

   }

   if (_head != NULL) {

     gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head");

     ret = false;

   }

   if (!ret) {

     gclog_or_tty->print_cr("### YOUNG LIST does not seem empty");

   }

   return ret;

 }

 void

 YoungList::rs_length_sampling_init() {

   _sampled_rs_lengths = 0;

   _curr               = _head;

 }

 bool

 YoungList::rs_length_sampling_more() {

   return _curr != NULL;

 }

 void

 YoungList::rs_length_sampling_next() {

   assert( _curr != NULL, "invariant" );

   size_t rs_length = _curr->rem_set()->occupied();

   _sampled_rs_lengths += rs_length;

   // The current region may not yet have been added to the

   // incremental collection set (it gets added when it is

   // retired as the current allocation region).

   if (_curr->in_collection_set()) {

     // Update the collection set policy information for this region

     _g1h->g1_policy()->update_incremental_cset_info(_curr, rs_length);

   }

   _curr = _curr->get_next_young_region();

   if (_curr == NULL) {

     _last_sampled_rs_lengths = _sampled_rs_lengths;

     // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths);

   }

 }

 void

 YoungList::reset_auxilary_lists() {

   guarantee( is_empty(), "young list should be empty" );

   assert(check_list_well_formed(), "young list should be well formed");

   // Add survivor regions to SurvRateGroup.

   _g1h->g1_policy()->note_start_adding_survivor_regions();

   _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */);

   for (HeapRegion* curr = _survivor_head;

        curr != NULL;

        curr = curr->get_next_young_region()) {

     _g1h->g1_policy()->set_region_survivors(curr);

     // The region is a non-empty survivor so let's add it to

     // the incremental collection set for the next evacuation

     // pause.

     _g1h->g1_policy()->add_region_to_incremental_cset_rhs(curr);

   }

   _g1h->g1_policy()->note_stop_adding_survivor_regions();

   _head   = _survivor_head;

   _length = _survivor_length;

   if (_survivor_head != NULL) {

     assert(_survivor_tail != NULL, "cause it shouldn't be");

     assert(_survivor_length > 0, "invariant");

     _survivor_tail->set_next_young_region(NULL);

   }

   // Don't clear the survivor list handles until the start of

   // the next evacuation pause - we need it in order to re-tag

   // the survivor regions from this evacuation pause as 'young'

   // at the start of the next.

   _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */);

   assert(check_list_well_formed(), "young list should be well formed");

 }

 void YoungList::print() {

   HeapRegion* lists[] = {_head,   _survivor_head};

   const char* names[] = {"YOUNG", "SURVIVOR"};

   for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) {

     gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]);

     HeapRegion *curr = lists[list];

     if (curr == NULL)

       gclog_or_tty->print_cr("  empty");

     while (curr != NULL) {

       gclog_or_tty->print_cr("  [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, "

                              "age: %4d, y: %d, surv: %d",

                              curr->bottom(), curr->end(),

                              curr->top(),

                              curr->prev_top_at_mark_start(),

                              curr->next_top_at_mark_start(),

                              curr->top_at_conc_mark_count(),

                              curr->age_in_surv_rate_group_cond(),

                              curr->is_young(),

                              curr->is_survivor());

       curr = curr->get_next_young_region();

     }

   }

   gclog_or_tty->print_cr("");

 }

 void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr)

 {

   // Claim the right to put the region on the dirty cards region list

   // by installing a self pointer.

   HeapRegion* next = hr->get_next_dirty_cards_region();

   if (next == NULL) {

     HeapRegion* res = (HeapRegion*)

       Atomic::cmpxchg_ptr(hr, hr->next_dirty_cards_region_addr(),

                           NULL);

     if (res == NULL) {

       HeapRegion* head;

       do {

         // Put the region to the dirty cards region list.

         head = _dirty_cards_region_list;

         next = (HeapRegion*)

           Atomic::cmpxchg_ptr(hr, &_dirty_cards_region_list, head);

         if (next == head) {

           assert(hr->get_next_dirty_cards_region() == hr,

                  "hr->get_next_dirty_cards_region() != hr");

           if (next == NULL) {

             // The last region in the list points to itself.

             hr->set_next_dirty_cards_region(hr);

           } else {

             hr->set_next_dirty_cards_region(next);

           }

         }

       } while (next != head);

     }

   }

 }

 HeapRegion* G1CollectedHeap::pop_dirty_cards_region()

 {

   HeapRegion* head;

   HeapRegion* hr;

   do {

     head = _dirty_cards_region_list;

     if (head == NULL) {

       return NULL;

     }

     HeapRegion* new_head = head->get_next_dirty_cards_region();

     if (head == new_head) {

       // The last region.

       new_head = NULL;

     }

     hr = (HeapRegion*)Atomic::cmpxchg_ptr(new_head, &_dirty_cards_region_list,

                                           head);

   } while (hr != head);

   assert(hr != NULL, "invariant");

   hr->set_next_dirty_cards_region(NULL);

   return hr;

 }

 void G1CollectedHeap::stop_conc_gc_threads() {

   _cg1r->stop();

   _cmThread->stop();

 }

 void G1CollectedHeap::check_ct_logs_at_safepoint() {

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set();

   // Count the dirty cards at the start.

   CountNonCleanMemRegionClosure count1(this);

   ct_bs->mod_card_iterate(&count1);

   int orig_count = count1.n();

   // First clear the logged cards.

   ClearLoggedCardTableEntryClosure clear;

   dcqs.set_closure(&clear);

   dcqs.apply_closure_to_all_completed_buffers();

   dcqs.iterate_closure_all_threads(false);

   clear.print_histo();

   // Now ensure that there's no dirty cards.

   CountNonCleanMemRegionClosure count2(this);

   ct_bs->mod_card_iterate(&count2);

   if (count2.n() != 0) {

     gclog_or_tty->print_cr("Card table has %d entries; %d originally",

                            count2.n(), orig_count);

   }

   guarantee(count2.n() == 0, "Card table should be clean.");

   RedirtyLoggedCardTableEntryClosure redirty;

   JavaThread::dirty_card_queue_set().set_closure(&redirty);

   dcqs.apply_closure_to_all_completed_buffers();

   dcqs.iterate_closure_all_threads(false);

   gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.",

                          clear.calls(), orig_count);

   guarantee(redirty.calls() == clear.calls(),

             "Or else mechanism is broken.");

   CountNonCleanMemRegionClosure count3(this);

   ct_bs->mod_card_iterate(&count3);

   if (count3.n() != orig_count) {

     gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.",

                            orig_count, count3.n());

     guarantee(count3.n() >= orig_count, "Should have restored them all.");

   }

   JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

 }

 // Private class members.

 G1CollectedHeap* G1CollectedHeap::_g1h;

 // Private methods.

 HeapRegion*

 G1CollectedHeap::new_region_try_secondary_free_list(size_t word_size) {

   MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);

   while (!_secondary_free_list.is_empty() || free_regions_coming()) {

     if (!_secondary_free_list.is_empty()) {

       if (G1ConcRegionFreeingVerbose) {

         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "

                                "secondary_free_list has "SIZE_FORMAT" entries",

                                _secondary_free_list.length());

       }

       // It looks as if there are free regions available on the

       // secondary_free_list. Let's move them to the free_list and try

       // again to allocate from it.

       append_secondary_free_list();

       assert(!_free_list.is_empty(), "if the secondary_free_list was not "

              "empty we should have moved at least one entry to the free_list");

       HeapRegion* res = _free_list.remove_head();

       if (G1ConcRegionFreeingVerbose) {

         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "

                                "allocated "HR_FORMAT" from secondary_free_list",

                                HR_FORMAT_PARAMS(res));

       }

       return res;

     }

     // Wait here until we get notifed either when (a) there are no

     // more free regions coming or (b) some regions have been moved on

     // the secondary_free_list.

     SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag);

   }

   if (G1ConcRegionFreeingVerbose) {

     gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "

                            "could not allocate from secondary_free_list");

   }

   return NULL;

 }

 HeapRegion* G1CollectedHeap::new_region_work(size_t word_size,

                                              bool do_expand) {

   assert(!isHumongous(word_size) ||

                                   word_size <= (size_t) HeapRegion::GrainWords,

          "the only time we use this to allocate a humongous region is "

          "when we are allocating a single humongous region");

   HeapRegion* res;

   if (G1StressConcRegionFreeing) {

     if (!_secondary_free_list.is_empty()) {

       if (G1ConcRegionFreeingVerbose) {

         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "

                                "forced to look at the secondary_free_list");

       }

       res = new_region_try_secondary_free_list(word_size);

       if (res != NULL) {

         return res;

       }

     }

   }

   res = _free_list.remove_head_or_null();

   if (res == NULL) {

     if (G1ConcRegionFreeingVerbose) {

       gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "

                              "res == NULL, trying the secondary_free_list");

     }

     res = new_region_try_secondary_free_list(word_size);

   }

   if (res == NULL && do_expand) {

     if (expand(word_size * HeapWordSize)) {

       // The expansion succeeded and so we should have at least one

       // region on the free list.

       res = _free_list.remove_head();

     }

   }

   if (res != NULL) {

     if (G1PrintHeapRegions) {

       gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT","PTR_FORMAT"], "

                              "top "PTR_FORMAT, res->hrs_index(),

                              res->bottom(), res->end(), res->top());

     }

   }

   return res;

 }

 HeapRegion* G1CollectedHeap::new_gc_alloc_region(int purpose,

                                                  size_t word_size) {

   HeapRegion* alloc_region = NULL;

   if (_gc_alloc_region_counts[purpose] < g1_policy()->max_regions(purpose)) {

     alloc_region = new_region_work(word_size, true /* do_expand */);

     if (purpose == GCAllocForSurvived && alloc_region != NULL) {

       alloc_region->set_survivor();

     }

     ++_gc_alloc_region_counts[purpose];

   } else {

     g1_policy()->note_alloc_region_limit_reached(purpose);

   }

   return alloc_region;

 }

 int G1CollectedHeap::humongous_obj_allocate_find_first(size_t num_regions,

                                                        size_t word_size) {

   int first = -1;

   if (num_regions == 1) {

     // Only one region to allocate, no need to go through the slower

     // path. The caller will attempt the expasion if this fails, so

     // let's not try to expand here too.

     HeapRegion* hr = new_region_work(word_size, false /* do_expand */);

     if (hr != NULL) {

       first = hr->hrs_index();

     } else {

       first = -1;

     }

   } else {

     // We can't allocate humongous regions while cleanupComplete() is

     // running, since some of the regions we find to be empty might not

     // yet be added to the free list and it is not straightforward to

     // know which list they are on so that we can remove them. Note

     // that we only need to do this if we need to allocate more than

     // one region to satisfy the current humongous allocation

     // request. If we are only allocating one region we use the common

     // region allocation code (see above).

     wait_while_free_regions_coming();

     append_secondary_free_list_if_not_empty();

     if (free_regions() >= num_regions) {

       first = _hrs->find_contiguous(num_regions);

       if (first != -1) {

         for (int i = first; i < first + (int) num_regions; ++i) {

           HeapRegion* hr = _hrs->at(i);

           assert(hr->is_empty(), "sanity");

           assert(is_on_free_list(hr), "sanity");

           hr->set_pending_removal(true);

         }

         _free_list.remove_all_pending(num_regions);

       }

     }

   }

   return first;

 }

 // If could fit into free regions w/o expansion, try.

 // Otherwise, if can expand, do so.

 // Otherwise, if using ex regions might help, try with ex given back.

 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size) {

   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);

   verify_region_sets_optional();

   size_t num_regions =

          round_to(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords;

   size_t x_size = expansion_regions();

   size_t fs = _hrs->free_suffix();

   int first = humongous_obj_allocate_find_first(num_regions, word_size);

   if (first == -1) {

     // The only thing we can do now is attempt expansion.

     if (fs + x_size >= num_regions) {

       // If the number of regions we're trying to allocate for this

       // object is at most the number of regions in the free suffix,

       // then the call to humongous_obj_allocate_find_first() above

       // should have succeeded and we wouldn't be here.

       //

       // We should only be trying to expand when the free suffix is

       // not sufficient for the object _and_ we have some expansion

       // room available.

       assert(num_regions > fs, "earlier allocation should have succeeded");

       if (expand((num_regions - fs) * HeapRegion::GrainBytes)) {

         first = humongous_obj_allocate_find_first(num_regions, word_size);

         // If the expansion was successful then the allocation

         // should have been successful.

         assert(first != -1, "this should have worked");

       }

     }

   }

   if (first != -1) {

     // Index of last region in the series + 1.

     int last = first + (int) num_regions;

     // We need to initialize the region(s) we just discovered. This is

     // a bit tricky given that it can happen concurrently with

     // refinement threads refining cards on these regions and

     // potentially wanting to refine the BOT as they are scanning

     // those cards (this can happen shortly after a cleanup; see CR

     // 6991377). So we have to set up the region(s) carefully and in

     // a specific order.

     // The word size sum of all the regions we will allocate.

     size_t word_size_sum = num_regions * HeapRegion::GrainWords;

     assert(word_size <= word_size_sum, "sanity");

     // This will be the "starts humongous" region.

     HeapRegion* first_hr = _hrs->at(first);

     // The header of the new object will be placed at the bottom of

     // the first region.

     HeapWord* new_obj = first_hr->bottom();

     // This will be the new end of the first region in the series that

     // should also match the end of the last region in the seriers.

     HeapWord* new_end = new_obj + word_size_sum;

     // This will be the new top of the first region that will reflect

     // this allocation.

     HeapWord* new_top = new_obj + word_size;

     // First, we need to zero the header of the space that we will be

     // allocating. When we update top further down, some refinement

     // threads might try to scan the region. By zeroing the header we

     // ensure that any thread that will try to scan the region will

     // come across the zero klass word and bail out.

     //

     // NOTE: It would not have been correct to have used

     // CollectedHeap::fill_with_object() and make the space look like

     // an int array. The thread that is doing the allocation will

     // later update the object header to a potentially different array

     // type and, for a very short period of time, the klass and length

     // fields will be inconsistent. This could cause a refinement

     // thread to calculate the object size incorrectly.

     Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);

     // We will set up the first region as "starts humongous". This

     // will also update the BOT covering all the regions to reflect

     // that there is a single object that starts at the bottom of the

     // first region.

     first_hr->set_startsHumongous(new_top, new_end);

     // Then, if there are any, we will set up the "continues

     // humongous" regions.

     HeapRegion* hr = NULL;

     for (int i = first + 1; i < last; ++i) {

       hr = _hrs->at(i);

       hr->set_continuesHumongous(first_hr);

     }

     // If we have "continues humongous" regions (hr != NULL), then the

     // end of the last one should match new_end.

     assert(hr == NULL || hr->end() == new_end, "sanity");

     // Up to this point no concurrent thread would have been able to

     // do any scanning on any region in this series. All the top

     // fields still point to bottom, so the intersection between

     // [bottom,top] and [card_start,card_end] will be empty. Before we

     // update the top fields, we'll do a storestore to make sure that

     // no thread sees the update to top before the zeroing of the

     // object header and the BOT initialization.

     OrderAccess::storestore();

     // Now that the BOT and the object header have been initialized,

     // we can update top of the "starts humongous" region.

     assert(first_hr->bottom() < new_top && new_top <= first_hr->end(),

            "new_top should be in this region");

     first_hr->set_top(new_top);

     // Now, we will update the top fields of the "continues humongous"

     // regions. The reason we need to do this is that, otherwise,

     // these regions would look empty and this will confuse parts of

     // G1. For example, the code that looks for a consecutive number

     // of empty regions will consider them empty and try to

     // re-allocate them. We can extend is_empty() to also include

     // !continuesHumongous(), but it is easier to just update the top

     // fields here. The way we set top for all regions (i.e., top ==

     // end for all regions but the last one, top == new_top for the

     // last one) is actually used when we will free up the humongous

     // region in free_humongous_region().

     hr = NULL;

     for (int i = first + 1; i < last; ++i) {

       hr = _hrs->at(i);

       if ((i + 1) == last) {

         // last continues humongous region

         assert(hr->bottom() < new_top && new_top <= hr->end(),

                "new_top should fall on this region");

         hr->set_top(new_top);

       } else {

         // not last one

         assert(new_top > hr->end(), "new_top should be above this region");

         hr->set_top(hr->end());

       }

     }

     // If we have continues humongous regions (hr != NULL), then the

     // end of the last one should match new_end and its top should

     // match new_top.

     assert(hr == NULL ||

            (hr->end() == new_end && hr->top() == new_top), "sanity");

     assert(first_hr->used() == word_size * HeapWordSize, "invariant");

     _summary_bytes_used += first_hr->used();

     _humongous_set.add(first_hr);

     return new_obj;

   }

   verify_region_sets_optional();

   return NULL;

 }

 void

 G1CollectedHeap::retire_cur_alloc_region(HeapRegion* cur_alloc_region) {

   // Other threads might still be trying to allocate using CASes out

   // of the region we are retiring, as they can do so without holding

   // the Heap_lock. So we first have to make sure that noone else can

   // allocate in it by doing a maximal allocation. Even if our CAS

   // attempt fails a few times, we'll succeed sooner or later given

   // that a failed CAS attempt mean that the region is getting closed

   // to being full (someone else succeeded in allocating into it).

   size_t free_word_size = cur_alloc_region->free() / HeapWordSize;

   // This is the minimum free chunk we can turn into a dummy

   // object. If the free space falls below this, then noone can

   // allocate in this region anyway (all allocation requests will be

   // of a size larger than this) so we won't have to perform the dummy

   // allocation.

   size_t min_word_size_to_fill = CollectedHeap::min_fill_size();

   while (free_word_size >= min_word_size_to_fill) {

     HeapWord* dummy =

       cur_alloc_region->par_allocate_no_bot_updates(free_word_size);

     if (dummy != NULL) {

       // If the allocation was successful we should fill in the space.

       CollectedHeap::fill_with_object(dummy, free_word_size);

       break;

     }

     free_word_size = cur_alloc_region->free() / HeapWordSize;

     // It's also possible that someone else beats us to the

     // allocation and they fill up the region. In that case, we can

     // just get out of the loop

   }

   assert(cur_alloc_region->free() / HeapWordSize < min_word_size_to_fill,

          "sanity");

   retire_cur_alloc_region_common(cur_alloc_region);

   assert(_cur_alloc_region == NULL, "post-condition");

 }

 // See the comment in the .hpp file about the locking protocol and

 // assumptions of this method (and other related ones).

 HeapWord*

 G1CollectedHeap::replace_cur_alloc_region_and_allocate(size_t word_size,

                                                        bool at_safepoint,

                                                        bool do_dirtying,

                                                        bool can_expand) {

   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);

   assert(_cur_alloc_region == NULL,

          "replace_cur_alloc_region_and_allocate() should only be called "

          "after retiring the previous current alloc region");

   assert(SafepointSynchronize::is_at_safepoint() == at_safepoint,

          "at_safepoint and is_at_safepoint() should be a tautology");

   assert(!can_expand || g1_policy()->can_expand_young_list(),

          "we should not call this method with can_expand == true if "

          "we are not allowed to expand the young gen");

   if (can_expand || !g1_policy()->is_young_list_full()) {

     HeapRegion* new_cur_alloc_region = new_alloc_region(word_size);

     if (new_cur_alloc_region != NULL) {

       assert(new_cur_alloc_region->is_empty(),

              "the newly-allocated region should be empty, "

              "as right now we only allocate new regions out of the free list");

       g1_policy()->update_region_num(true /* next_is_young */);

       set_region_short_lived_locked(new_cur_alloc_region);

       assert(!new_cur_alloc_region->isHumongous(),

              "Catch a regression of this bug.");

       // We need to ensure that the stores to _cur_alloc_region and,

       // subsequently, to top do not float above the setting of the

       // young type.

       OrderAccess::storestore();

       // Now, perform the allocation out of the region we just

       // allocated. Note that noone else can access that region at

       // this point (as _cur_alloc_region has not been updated yet),

       // so we can just go ahead and do the allocation without any

       // atomics (and we expect this allocation attempt to

       // suceeded). Given that other threads can attempt an allocation

       // with a CAS and without needing the Heap_lock, if we assigned

       // the new region to _cur_alloc_region before first allocating

       // into it other threads might have filled up the new region

       // before we got a chance to do the allocation ourselves. In

       // that case, we would have needed to retire the region, grab a

       // new one, and go through all this again. Allocating out of the

       // new region before assigning it to _cur_alloc_region avoids

       // all this.

       HeapWord* result =

                      new_cur_alloc_region->allocate_no_bot_updates(word_size);

       assert(result != NULL, "we just allocate out of an empty region "

              "so allocation should have been successful");

       assert(is_in(result), "result should be in the heap");

       // Now make sure that the store to _cur_alloc_region does not

       // float above the store to top.

       OrderAccess::storestore();

       _cur_alloc_region = new_cur_alloc_region;

       if (!at_safepoint) {

         Heap_lock->unlock();

       }

       // do the dirtying, if necessary, after we release the Heap_lock

       if (do_dirtying) {

         dirty_young_block(result, word_size);

       }

       return result;

     }

   }

   assert(_cur_alloc_region == NULL, "we failed to allocate a new current "

          "alloc region, it should still be NULL");

   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);

   return NULL;

 }

 // See the comment in the .hpp file about the locking protocol and

 // assumptions of this method (and other related ones).

 HeapWord*

 G1CollectedHeap::attempt_allocation_slow(size_t word_size) {

   assert_heap_locked_and_not_at_safepoint();

   assert(!isHumongous(word_size), "attempt_allocation_slow() should not be "

          "used for humongous allocations");

   // We should only reach here when we were unable to allocate

   // otherwise. So, we should have not active current alloc region.

   assert(_cur_alloc_region == NULL, "current alloc region should be NULL");

   // We will loop while succeeded is false, which means that we tried

   // to do a collection, but the VM op did not succeed. So, when we

   // exit the loop, either one of the allocation attempts was

   // successful, or we succeeded in doing the VM op but which was

   // unable to allocate after the collection.

   for (int try_count = 1; /* we'll return or break */; try_count += 1) {

     bool succeeded = true;

     // Every time we go round the loop we should be holding the Heap_lock.

     assert_heap_locked();

     if (GC_locker::is_active_and_needs_gc()) {

       // We are locked out of GC because of the GC locker. We can

       // allocate a new region only if we can expand the young gen.

       if (g1_policy()->can_expand_young_list()) {

         // Yes, we are allowed to expand the young gen. Let's try to

         // allocate a new current alloc region.

         HeapWord* result =

           replace_cur_alloc_region_and_allocate(word_size,

                                                 false, /* at_safepoint */

                                                 true,  /* do_dirtying */

                                                 true   /* can_expand */);

         if (result != NULL) {

           assert_heap_not_locked();

           return result;

         }

       }

       // We could not expand the young gen further (or we could but we

       // failed to allocate a new region). We'll stall until the GC

       // locker forces a GC.

       // If this thread is not in a jni critical section, we stall

       // the requestor until the critical section has cleared and

       // GC allowed. When the critical section clears, a GC is

       // initiated by the last thread exiting the critical section; so

       // we retry the allocation sequence from the beginning of the loop,

       // rather than causing more, now probably unnecessary, GC attempts.

       JavaThread* jthr = JavaThread::current();

       assert(jthr != NULL, "sanity");

       if (jthr->in_critical()) {

         if (CheckJNICalls) {

           fatal("Possible deadlock due to allocating while"

                 " in jni critical section");

         }

         // We are returning NULL so the protocol is that we're still

         // holding the Heap_lock.

         assert_heap_locked();

         return NULL;

       }

       Heap_lock->unlock();

       GC_locker::stall_until_clear();

       // No need to relock the Heap_lock. We'll fall off to the code

       // below the else-statement which assumes that we are not

       // holding the Heap_lock.

     } else {

       // We are not locked out. So, let's try to do a GC. The VM op

       // will retry the allocation before it completes.

       // Read the GC count while holding the Heap_lock

       unsigned int gc_count_before = SharedHeap::heap()->total_collections();

       Heap_lock->unlock();

       HeapWord* result =

         do_collection_pause(word_size, gc_count_before, &succeeded);

       assert_heap_not_locked();

       if (result != NULL) {

         assert(succeeded, "the VM op should have succeeded");

         // Allocations that take place on VM operations do not do any

         // card dirtying and we have to do it here.

         dirty_young_block(result, word_size);

         return result;

       }

     }

     // Both paths that get us here from above unlock the Heap_lock.

     assert_heap_not_locked();

     // We can reach here when we were unsuccessful in doing a GC,

     // because another thread beat us to it, or because we were locked

     // out of GC due to the GC locker. In either case a new alloc

     // region might be available so we will retry the allocation.

     HeapWord* result = attempt_allocation(word_size);

     if (result != NULL) {

       assert_heap_not_locked();

       return result;

     }

     // So far our attempts to allocate failed. The only time we'll go

     // around the loop and try again is if we tried to do a GC and the

     // VM op that we tried to schedule was not successful because

     // another thread beat us to it. If that happened it's possible

     // that by the time we grabbed the Heap_lock again and tried to

     // allocate other threads filled up the young generation, which

     // means that the allocation attempt after the GC also failed. So,

     // it's worth trying to schedule another GC pause.

     if (succeeded) {

       break;

     }

     // Give a warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::attempt_allocation_slow() "

               "retries %d times", try_count);

     }

   }

   assert_heap_locked();

   return NULL;

 }

 // See the comment in the .hpp file about the locking protocol and

 // assumptions of this method (and other related ones).

 HeapWord*

 G1CollectedHeap::attempt_allocation_humongous(size_t word_size,

                                               bool at_safepoint) {

   // This is the method that will allocate a humongous object. All

   // allocation paths that attempt to allocate a humongous object

   // should eventually reach here. Currently, the only paths are from

   // mem_allocate() and attempt_allocation_at_safepoint().

   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);

   assert(isHumongous(word_size), "attempt_allocation_humongous() "

          "should only be used for humongous allocations");

   assert(SafepointSynchronize::is_at_safepoint() == at_safepoint,

          "at_safepoint and is_at_safepoint() should be a tautology");

   HeapWord* result = NULL;

   // We will loop while succeeded is false, which means that we tried

   // to do a collection, but the VM op did not succeed. So, when we

   // exit the loop, either one of the allocation attempts was

   // successful, or we succeeded in doing the VM op but which was

   // unable to allocate after the collection.

   for (int try_count = 1; /* we'll return or break */; try_count += 1) {

     bool succeeded = true;

     // Given that humongous objects are not allocated in young

     // regions, we'll first try to do the allocation without doing a

     // collection hoping that there's enough space in the heap.

     result = humongous_obj_allocate(word_size);

     assert(_cur_alloc_region == NULL || !_cur_alloc_region->isHumongous(),

            "catch a regression of this bug.");

     if (result != NULL) {

       if (!at_safepoint) {

         // If we're not at a safepoint, unlock the Heap_lock.

         Heap_lock->unlock();

       }

       return result;

     }

     // If we failed to allocate the humongous object, we should try to

     // do a collection pause (if we're allowed) in case it reclaims

     // enough space for the allocation to succeed after the pause.

     if (!at_safepoint) {

       // Read the GC count while holding the Heap_lock

       unsigned int gc_count_before = SharedHeap::heap()->total_collections();

       // If we're allowed to do a collection we're not at a

       // safepoint, so it is safe to unlock the Heap_lock.

       Heap_lock->unlock();

       result = do_collection_pause(word_size, gc_count_before, &succeeded);

       assert_heap_not_locked();

       if (result != NULL) {

         assert(succeeded, "the VM op should have succeeded");

         return result;

       }

       // If we get here, the VM operation either did not succeed

       // (i.e., another thread beat us to it) or it succeeded but

       // failed to allocate the object.

       // If we're allowed to do a collection we're not at a

       // safepoint, so it is safe to lock the Heap_lock.

       Heap_lock->lock();

     }

     assert(result == NULL, "otherwise we should have exited the loop earlier");

     // So far our attempts to allocate failed. The only time we'll go

     // around the loop and try again is if we tried to do a GC and the

     // VM op that we tried to schedule was not successful because

     // another thread beat us to it. That way it's possible that some

     // space was freed up by the thread that successfully scheduled a

     // GC. So it's worth trying to allocate again.

     if (succeeded) {

       break;

     }

     // Give a warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::attempt_allocation_humongous "

               "retries %d times", try_count);

     }

   }

   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);

   return NULL;

 }

 HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size,

                                            bool expect_null_cur_alloc_region) {

   assert_at_safepoint(true /* should_be_vm_thread */);

   assert(_cur_alloc_region == NULL || !expect_null_cur_alloc_region,

          err_msg("the current alloc region was unexpectedly found "

                  "to be non-NULL, cur alloc region: "PTR_FORMAT" "

                  "expect_null_cur_alloc_region: %d word_size: "SIZE_FORMAT,

                  _cur_alloc_region, expect_null_cur_alloc_region, word_size));

   if (!isHumongous(word_size)) {

     if (!expect_null_cur_alloc_region) {

       HeapRegion* cur_alloc_region = _cur_alloc_region;

       if (cur_alloc_region != NULL) {

         // We are at a safepoint so no reason to use the MT-safe version.

         HeapWord* result = cur_alloc_region->allocate_no_bot_updates(word_size);

         if (result != NULL) {

           assert(is_in(result), "result should be in the heap");

           // We will not do any dirtying here. This is guaranteed to be

           // called during a safepoint and the thread that scheduled the

           // pause will do the dirtying if we return a non-NULL result.

           return result;

         }

         retire_cur_alloc_region_common(cur_alloc_region);

       }

     }

     assert(_cur_alloc_region == NULL,

            "at this point we should have no cur alloc region");

     return replace_cur_alloc_region_and_allocate(word_size,

                                                  true, /* at_safepoint */

                                                  false /* do_dirtying */,

                                                  false /* can_expand */);

   } else {

     return attempt_allocation_humongous(word_size,

                                         true /* at_safepoint */);

   }

   ShouldNotReachHere();

 }

 HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) {

   assert_heap_not_locked_and_not_at_safepoint();

   assert(!isHumongous(word_size), "we do not allow TLABs of humongous size");

   // First attempt: Try allocating out of the current alloc region

   // using a CAS. If that fails, take the Heap_lock and retry the

   // allocation, potentially replacing the current alloc region.

   HeapWord* result = attempt_allocation(word_size);

   if (result != NULL) {

     assert_heap_not_locked();

     return result;

   }

   // Second attempt: Go to the slower path where we might try to

   // schedule a collection.

   result = attempt_allocation_slow(word_size);

   if (result != NULL) {

     assert_heap_not_locked();

     return result;

   }

   assert_heap_locked();

   // Need to unlock the Heap_lock before returning.

   Heap_lock->unlock();

   return NULL;

 }

 HeapWord*

 G1CollectedHeap::mem_allocate(size_t word_size,

                               bool   is_noref,

                               bool   is_tlab,

                               bool*  gc_overhead_limit_was_exceeded) {

   assert_heap_not_locked_and_not_at_safepoint();

   assert(!is_tlab, "mem_allocate() this should not be called directly "

          "to allocate TLABs");

   // Loop until the allocation is satisified,

   // or unsatisfied after GC.

   for (int try_count = 1; /* we'll return */; try_count += 1) {

     unsigned int gc_count_before;

     {

       if (!isHumongous(word_size)) {

         // First attempt: Try allocating out of the current alloc region

         // using a CAS. If that fails, take the Heap_lock and retry the

         // allocation, potentially replacing the current alloc region.

         HeapWord* result = attempt_allocation(word_size);

         if (result != NULL) {

           assert_heap_not_locked();

           return result;

         }

         assert_heap_locked();

         // Second attempt: Go to the slower path where we might try to

         // schedule a collection.

         result = attempt_allocation_slow(word_size);

         if (result != NULL) {

           assert_heap_not_locked();

           return result;

         }

       } else {

         // attempt_allocation_humongous() requires the Heap_lock to be held.

         Heap_lock->lock();

         HeapWord* result = attempt_allocation_humongous(word_size,

                                                      false /* at_safepoint */);

         if (result != NULL) {

           assert_heap_not_locked();

           return result;

         }

       }

       assert_heap_locked();

       // Read the gc count while the heap lock is held.

       gc_count_before = SharedHeap::heap()->total_collections();

       // Release the Heap_lock before attempting the collection.

       Heap_lock->unlock();

     }

     // Create the garbage collection operation...

     VM_G1CollectForAllocation op(gc_count_before, word_size);

     // ...and get the VM thread to execute it.

     VMThread::execute(&op);

     assert_heap_not_locked();

     if (op.prologue_succeeded() && op.pause_succeeded()) {

       // If the operation was successful we'll return the result even

       // if it is NULL. If the allocation attempt failed immediately

       // after a Full GC, it's unlikely we'll be able to allocate now.

       HeapWord* result = op.result();

       if (result != NULL && !isHumongous(word_size)) {

         // Allocations that take place on VM operations do not do any

         // card dirtying and we have to do it here. We only have to do

         // this for non-humongous allocations, though.

         dirty_young_block(result, word_size);

       }

       return result;

     } else {

       assert(op.result() == NULL,

              "the result should be NULL if the VM op did not succeed");

     }

     // Give a warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::mem_allocate retries %d times", try_count);

     }

   }

   ShouldNotReachHere();

 }

 void G1CollectedHeap::abandon_cur_alloc_region() {

   assert_at_safepoint(true /* should_be_vm_thread */);

   HeapRegion* cur_alloc_region = _cur_alloc_region;

   if (cur_alloc_region != NULL) {

     assert(!cur_alloc_region->is_empty(),

            "the current alloc region can never be empty");

     assert(cur_alloc_region->is_young(),

            "the current alloc region should be young");

     retire_cur_alloc_region_common(cur_alloc_region);

   }

   assert(_cur_alloc_region == NULL, "post-condition");

 }

 void G1CollectedHeap::abandon_gc_alloc_regions() {

   // first, make sure that the GC alloc region list is empty (it should!)

   assert(_gc_alloc_region_list == NULL, "invariant");

   release_gc_alloc_regions(true /* totally */);

 }

 class PostMCRemSetClearClosure: public HeapRegionClosure {

   ModRefBarrierSet* _mr_bs;

 public:

   PostMCRemSetClearClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}

   bool doHeapRegion(HeapRegion* r) {

     r->reset_gc_time_stamp();

     if (r->continuesHumongous())

       return false;

     HeapRegionRemSet* hrrs = r->rem_set();

     if (hrrs != NULL) hrrs->clear();

     // You might think here that we could clear just the cards

     // corresponding to the used region.  But no: if we leave a dirty card

     // in a region we might allocate into, then it would prevent that card

     // from being enqueued, and cause it to be missed.

     // Re: the performance cost: we shouldn't be doing full GC anyway!

     _mr_bs->clear(MemRegion(r->bottom(), r->end()));

     return false;

   }

 };

 class PostMCRemSetInvalidateClosure: public HeapRegionClosure {

   ModRefBarrierSet* _mr_bs;

 public:

   PostMCRemSetInvalidateClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}

   bool doHeapRegion(HeapRegion* r) {

     if (r->continuesHumongous()) return false;

     if (r->used_region().word_size() != 0) {

       _mr_bs->invalidate(r->used_region(), true /*whole heap*/);

     }

     return false;

   }

 };

 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {

   G1CollectedHeap*   _g1h;

   UpdateRSOopClosure _cl;

   int                _worker_i;

 public:

   RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, int worker_i = 0) :

     _cl(g1->g1_rem_set(), worker_i),

     _worker_i(worker_i),

     _g1h(g1)

   { }

   bool doHeapRegion(HeapRegion* r) {

     if (!r->continuesHumongous()) {

       _cl.set_from(r);

       r->oop_iterate(&_cl);

     }

     return false;

   }

 };

 class ParRebuildRSTask: public AbstractGangTask {

   G1CollectedHeap* _g1;

 public:

   ParRebuildRSTask(G1CollectedHeap* g1)

     : AbstractGangTask("ParRebuildRSTask"),

       _g1(g1)

   { }

   void work(int i) {

     RebuildRSOutOfRegionClosure rebuild_rs(_g1, i);

     _g1->heap_region_par_iterate_chunked(&rebuild_rs, i,

                                          HeapRegion::RebuildRSClaimValue);

   }

 };

 bool G1CollectedHeap::do_collection(bool explicit_gc,

                                     bool clear_all_soft_refs,

                                     size_t word_size) {

   assert_at_safepoint(true /* should_be_vm_thread */);

   if (GC_locker::check_active_before_gc()) {

     return false;

   }

   SvcGCMarker sgcm(SvcGCMarker::FULL);

   ResourceMark rm;

   if (PrintHeapAtGC) {

     Universe::print_heap_before_gc();

   }

   verify_region_sets_optional();

   const bool do_clear_all_soft_refs = clear_all_soft_refs ||

                            collector_policy()->should_clear_all_soft_refs();

   ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy());

   {

     IsGCActiveMark x;

     // Timing

     bool system_gc = (gc_cause() == GCCause::_java_lang_system_gc);

     assert(!system_gc || explicit_gc, "invariant");

     gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);

     TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);

     TraceTime t(system_gc ? "Full GC (System.gc())" : "Full GC",

                 PrintGC, true, gclog_or_tty);

     TraceMemoryManagerStats tms(true /* fullGC */);

     double start = os::elapsedTime();

     g1_policy()->record_full_collection_start();

     wait_while_free_regions_coming();

     append_secondary_free_list_if_not_empty();

     gc_prologue(true);

     increment_total_collections(true /* full gc */);

     size_t g1h_prev_used = used();

     assert(used() == recalculate_used(), "Should be equal");

     if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       prepare_for_verify();

       gclog_or_tty->print(" VerifyBeforeGC:");

       Universe::verify(true);

     }

     COMPILER2_PRESENT(DerivedPointerTable::clear());

     // We want to discover references, but not process them yet.

     // This mode is disabled in

     // instanceRefKlass::process_discovered_references if the

     // generation does some collection work, or

     // instanceRefKlass::enqueue_discovered_references if the

     // generation returns without doing any work.

     ref_processor()->disable_discovery();

     ref_processor()->abandon_partial_discovery();

     ref_processor()->verify_no_references_recorded();

     // Abandon current iterations of concurrent marking and concurrent

     // refinement, if any are in progress.

     concurrent_mark()->abort();

     // Make sure we'll choose a new allocation region afterwards.

     abandon_cur_alloc_region();

     abandon_gc_alloc_regions();

     assert(_cur_alloc_region == NULL, "Invariant.");

     g1_rem_set()->cleanupHRRS();

     tear_down_region_lists();

     // We may have added regions to the current incremental collection

     // set between the last GC or pause and now. We need to clear the

     // incremental collection set and then start rebuilding it afresh

     // after this full GC.

     abandon_collection_set(g1_policy()->inc_cset_head());

     g1_policy()->clear_incremental_cset();

     g1_policy()->stop_incremental_cset_building();

     if (g1_policy()->in_young_gc_mode()) {

       empty_young_list();

       g1_policy()->set_full_young_gcs(true);

     }

     // See the comment in G1CollectedHeap::ref_processing_init() about

     // how reference processing currently works in G1.

     // Temporarily make reference _discovery_ single threaded (non-MT).

     ReferenceProcessorMTMutator rp_disc_ser(ref_processor(), false);

     // Temporarily make refs discovery atomic

     ReferenceProcessorAtomicMutator rp_disc_atomic(ref_processor(), true);

     // Temporarily clear _is_alive_non_header

     ReferenceProcessorIsAliveMutator rp_is_alive_null(ref_processor(), NULL);

     ref_processor()->enable_discovery();

     ref_processor()->setup_policy(do_clear_all_soft_refs);

     // Do collection work

     {

       HandleMark hm;  // Discard invalid handles created during gc

       G1MarkSweep::invoke_at_safepoint(ref_processor(), do_clear_all_soft_refs);

     }

     assert(free_regions() == 0, "we should not have added any free regions");

     rebuild_region_lists();

     _summary_bytes_used = recalculate_used();

     ref_processor()->enqueue_discovered_references();

     COMPILER2_PRESENT(DerivedPointerTable::update_pointers());

     MemoryService::track_memory_usage();

     if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       gclog_or_tty->print(" VerifyAfterGC:");

       prepare_for_verify();

       Universe::verify(false);

     }

     NOT_PRODUCT(ref_processor()->verify_no_references_recorded());

     reset_gc_time_stamp();

     // Since everything potentially moved, we will clear all remembered

     // sets, and clear all cards.  Later we will rebuild remebered

     // sets. We will also reset the GC time stamps of the regions.

     PostMCRemSetClearClosure rs_clear(mr_bs());

     heap_region_iterate(&rs_clear);

     // Resize the heap if necessary.

     resize_if_necessary_after_full_collection(explicit_gc ? 0 : word_size);

     if (_cg1r->use_cache()) {

       _cg1r->clear_and_record_card_counts();

       _cg1r->clear_hot_cache();

     }

     // Rebuild remembered sets of all regions.

     if (G1CollectedHeap::use_parallel_gc_threads()) {

       ParRebuildRSTask rebuild_rs_task(this);

       assert(check_heap_region_claim_values(

              HeapRegion::InitialClaimValue), "sanity check");

       set_par_threads(workers()->total_workers());

       workers()->run_task(&rebuild_rs_task);

       set_par_threads(0);

       assert(check_heap_region_claim_values(

              HeapRegion::RebuildRSClaimValue), "sanity check");

       reset_heap_region_claim_values();

     } else {

       RebuildRSOutOfRegionClosure rebuild_rs(this);

       heap_region_iterate(&rebuild_rs);

     }

     if (PrintGC) {

       print_size_transition(gclog_or_tty, g1h_prev_used, used(), capacity());

     }

     if (true) { // FIXME

       // Ask the permanent generation to adjust size for full collections

       perm()->compute_new_size();

     }

     // Start a new incremental collection set for the next pause

     assert(g1_policy()->collection_set() == NULL, "must be");

     g1_policy()->start_incremental_cset_building();

     // Clear the _cset_fast_test bitmap in anticipation of adding

     // regions to the incremental collection set for the next

     // evacuation pause.

     clear_cset_fast_test();

     double end = os::elapsedTime();

     g1_policy()->record_full_collection_end();

 #ifdef TRACESPINNING

     ParallelTaskTerminator::print_termination_counts();

 #endif

     gc_epilogue(true);

     // Discard all rset updates

     JavaThread::dirty_card_queue_set().abandon_logs();

     assert(!G1DeferredRSUpdate

            || (G1DeferredRSUpdate && (dirty_card_queue_set().completed_buffers_num() == 0)), "Should not be any");

   }

   if (g1_policy()->in_young_gc_mode()) {

     _young_list->reset_sampled_info();

     // At this point there should be no regions in the

     // entire heap tagged as young.

     assert( check_young_list_empty(true /* check_heap */),

             "young list should be empty at this point");

   }

   // Update the number of full collections that have been completed.

   increment_full_collections_completed(false /* concurrent */);

   verify_region_sets_optional();

   if (PrintHeapAtGC) {

     Universe::print_heap_after_gc();

   }

   return true;

 }

 void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {

   // do_collection() will return whether it succeeded in performing

   // the GC. Currently, there is no facility on the

   // do_full_collection() API to notify the caller than the collection

   // did not succeed (e.g., because it was locked out by the GC

   // locker). So, right now, we'll ignore the return value.

   bool dummy = do_collection(true,                /* explicit_gc */

                              clear_all_soft_refs,

                              0                    /* word_size */);

 }

 // This code is mostly copied from TenuredGeneration.

 void

 G1CollectedHeap::

 resize_if_necessary_after_full_collection(size_t word_size) {

   assert(MinHeapFreeRatio <= MaxHeapFreeRatio, "sanity check");

   // Include the current allocation, if any, and bytes that will be

   // pre-allocated to support collections, as "used".

   const size_t used_after_gc = used();

   const size_t capacity_after_gc = capacity();

   const size_t free_after_gc = capacity_after_gc - used_after_gc;

   // This is enforced in arguments.cpp.

   assert(MinHeapFreeRatio <= MaxHeapFreeRatio,

          "otherwise the code below doesn't make sense");

   // We don't have floating point command-line arguments

   const double minimum_free_percentage = (double) MinHeapFreeRatio / 100.0;

   const double maximum_used_percentage = 1.0 - minimum_free_percentage;

   const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100.0;

   const double minimum_used_percentage = 1.0 - maximum_free_percentage;

   const size_t min_heap_size = collector_policy()->min_heap_byte_size();

   const size_t max_heap_size = collector_policy()->max_heap_byte_size();

   // We have to be careful here as these two calculations can overflow

   // 32-bit size_t's.

   double used_after_gc_d = (double) used_after_gc;

   double minimum_desired_capacity_d = used_after_gc_d / maximum_used_percentage;

   double maximum_desired_capacity_d = used_after_gc_d / minimum_used_percentage;

   // Let's make sure that they are both under the max heap size, which

   // by default will make them fit into a size_t.

   double desired_capacity_upper_bound = (double) max_heap_size;

   minimum_desired_capacity_d = MIN2(minimum_desired_capacity_d,

                                     desired_capacity_upper_bound);

   maximum_desired_capacity_d = MIN2(maximum_desired_capacity_d,

                                     desired_capacity_upper_bound);

   // We can now safely turn them into size_t's.

   size_t minimum_desired_capacity = (size_t) minimum_desired_capacity_d;

   size_t maximum_desired_capacity = (size_t) maximum_desired_capacity_d;

   // This assert only makes sense here, before we adjust them

   // with respect to the min and max heap size.

   assert(minimum_desired_capacity <= maximum_desired_capacity,

          err_msg("minimum_desired_capacity = "SIZE_FORMAT", "

                  "maximum_desired_capacity = "SIZE_FORMAT,

                  minimum_desired_capacity, maximum_desired_capacity));

   // Should not be greater than the heap max size. No need to adjust

   // it with respect to the heap min size as it's a lower bound (i.e.,

   // we'll try to make the capacity larger than it, not smaller).

   minimum_desired_capacity = MIN2(minimum_desired_capacity, max_heap_size);

   // Should not be less than the heap min size. No need to adjust it

   // with respect to the heap max size as it's an upper bound (i.e.,

   // we'll try to make the capacity smaller than it, not greater).

   maximum_desired_capacity =  MAX2(maximum_desired_capacity, min_heap_size);

   if (PrintGC && Verbose) {

     const double free_percentage =

       (double) free_after_gc / (double) capacity_after_gc;

     gclog_or_tty->print_cr("Computing new size after full GC ");

     gclog_or_tty->print_cr("  "

                            "  minimum_free_percentage: %6.2f",

                            minimum_free_percentage);

     gclog_or_tty->print_cr("  "

                            "  maximum_free_percentage: %6.2f",

                            maximum_free_percentage);

     gclog_or_tty->print_cr("  "

                            "  capacity: %6.1fK"

                            "  minimum_desired_capacity: %6.1fK"

                            "  maximum_desired_capacity: %6.1fK",

                            (double) capacity_after_gc / (double) K,

                            (double) minimum_desired_capacity / (double) K,

                            (double) maximum_desired_capacity / (double) K);

     gclog_or_tty->print_cr("  "

                            "  free_after_gc: %6.1fK"

                            "  used_after_gc: %6.1fK",

                            (double) free_after_gc / (double) K,

                            (double) used_after_gc / (double) K);

     gclog_or_tty->print_cr("  "

                            "   free_percentage: %6.2f",

                            free_percentage);

   }

   if (capacity_after_gc < minimum_desired_capacity) {

     // Don't expand unless it's significant

     size_t expand_bytes = minimum_desired_capacity - capacity_after_gc;

     if (expand(expand_bytes)) {

       if (PrintGC && Verbose) {

         gclog_or_tty->print_cr("  "

                                "  expanding:"

                                "  max_heap_size: %6.1fK"

                                "  minimum_desired_capacity: %6.1fK"

                                "  expand_bytes: %6.1fK",

                                (double) max_heap_size / (double) K,

                                (double) minimum_desired_capacity / (double) K,

                                (double) expand_bytes / (double) K);

       }

     }

     // No expansion, now see if we want to shrink

   } else if (capacity_after_gc > maximum_desired_capacity) {

     // Capacity too large, compute shrinking size

     size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity;

     shrink(shrink_bytes);

     if (PrintGC && Verbose) {

       gclog_or_tty->print_cr("  "

                              "  shrinking:"

                              "  min_heap_size: %6.1fK"

                              "  maximum_desired_capacity: %6.1fK"

                              "  shrink_bytes: %6.1fK",

                              (double) min_heap_size / (double) K,

                              (double) maximum_desired_capacity / (double) K,

                              (double) shrink_bytes / (double) K);

     }

   }

 }

 HeapWord*

 G1CollectedHeap::satisfy_failed_allocation(size_t word_size,

                                            bool* succeeded) {

   assert_at_safepoint(true /* should_be_vm_thread */);

   *succeeded = true;

   // Let's attempt the allocation first.

   HeapWord* result = attempt_allocation_at_safepoint(word_size,

                                      false /* expect_null_cur_alloc_region */);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   // In a G1 heap, we're supposed to keep allocation from failing by

   // incremental pauses.  Therefore, at least for now, we'll favor

   // expansion over collection.  (This might change in the future if we can

   // do something smarter than full collection to satisfy a failed alloc.)

   result = expand_and_allocate(word_size);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   // Expansion didn't work, we'll try to do a Full GC.

   bool gc_succeeded = do_collection(false, /* explicit_gc */

                                     false, /* clear_all_soft_refs */

                                     word_size);

   if (!gc_succeeded) {

     *succeeded = false;

     return NULL;

   }

   // Retry the allocation

   result = attempt_allocation_at_safepoint(word_size,

                                       true /* expect_null_cur_alloc_region */);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   // Then, try a Full GC that will collect all soft references.

   gc_succeeded = do_collection(false, /* explicit_gc */

                                true,  /* clear_all_soft_refs */

                                word_size);

   if (!gc_succeeded) {

     *succeeded = false;

     return NULL;

   }

   // Retry the allocation once more

   result = attempt_allocation_at_safepoint(word_size,

                                       true /* expect_null_cur_alloc_region */);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   assert(!collector_policy()->should_clear_all_soft_refs(),

          "Flag should have been handled and cleared prior to this point");

   // What else?  We might try synchronous finalization later.  If the total

   // space available is large enough for the allocation, then a more

   // complete compaction phase than we've tried so far might be

   // appropriate.

   assert(*succeeded, "sanity");

   return NULL;

 }

 // Attempting to expand the heap sufficiently

 // to support an allocation of the given "word_size".  If

 // successful, perform the allocation and return the address of the

 // allocated block, or else "NULL".

 HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size) {

   assert_at_safepoint(true /* should_be_vm_thread */);

   verify_region_sets_optional();

   size_t expand_bytes = MAX2(word_size * HeapWordSize, MinHeapDeltaBytes);

   if (expand(expand_bytes)) {

     verify_region_sets_optional();

     return attempt_allocation_at_safepoint(word_size,

                                           false /* expect_null_cur_alloc_region */);

   }

   return NULL;

 }

 bool G1CollectedHeap::expand(size_t expand_bytes) {

   size_t old_mem_size = _g1_storage.committed_size();

   size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes);

   aligned_expand_bytes = align_size_up(aligned_expand_bytes,

                                        HeapRegion::GrainBytes);

   if (Verbose && PrintGC) {

     gclog_or_tty->print("Expanding garbage-first heap from %ldK by %ldK",

                            old_mem_size/K, aligned_expand_bytes/K);

   }

   HeapWord* old_end = (HeapWord*)_g1_storage.high();

   bool successful = _g1_storage.expand_by(aligned_expand_bytes);

   if (successful) {

     HeapWord* new_end = (HeapWord*)_g1_storage.high();

     // Expand the committed region.

     _g1_committed.set_end(new_end);

     // Tell the cardtable about the expansion.

     Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);

     // And the offset table as well.

     _bot_shared->resize(_g1_committed.word_size());

     expand_bytes = aligned_expand_bytes;

     HeapWord* base = old_end;

     // Create the heap regions for [old_end, new_end)

     while (expand_bytes > 0) {

       HeapWord* high = base + HeapRegion::GrainWords;

       // Create a new HeapRegion.

       MemRegion mr(base, high);

       bool is_zeroed = !_g1_max_committed.contains(base);

       HeapRegion* hr = new HeapRegion(_bot_shared, mr, is_zeroed);

       // Add it to the HeapRegionSeq.

       _hrs->insert(hr);

       _free_list.add_as_tail(hr);

       // And we used up an expansion region to create it.

       _expansion_regions--;

       expand_bytes -= HeapRegion::GrainBytes;

       base += HeapRegion::GrainWords;

     }

     assert(base == new_end, "sanity");

     // Now update max_committed if necessary.

     _g1_max_committed.set_end(MAX2(_g1_max_committed.end(), new_end));

   } else {

     // The expansion of the virtual storage space was unsuccessful.

     // Let's see if it was because we ran out of swap.

     if (G1ExitOnExpansionFailure &&

         _g1_storage.uncommitted_size() >= aligned_expand_bytes) {

       // We had head room...

       vm_exit_out_of_memory(aligned_expand_bytes, "G1 heap expansion");

     }

   }

   if (Verbose && PrintGC) {

     size_t new_mem_size = _g1_storage.committed_size();

     gclog_or_tty->print_cr("...%s, expanded to %ldK",

                            (successful ? "Successful" : "Failed"),

                            new_mem_size/K);

   }

   return successful;

 }

 void G1CollectedHeap::shrink_helper(size_t shrink_bytes)

 {

   size_t old_mem_size = _g1_storage.committed_size();

   size_t aligned_shrink_bytes =

     ReservedSpace::page_align_size_down(shrink_bytes);

   aligned_shrink_bytes = align_size_down(aligned_shrink_bytes,

                                          HeapRegion::GrainBytes);

   size_t num_regions_deleted = 0;

   MemRegion mr = _hrs->shrink_by(aligned_shrink_bytes, num_regions_deleted);

   assert(mr.end() == (HeapWord*)_g1_storage.high(), "Bad shrink!");

   if (mr.byte_size() > 0)

     _g1_storage.shrink_by(mr.byte_size());

   assert(mr.start() == (HeapWord*)_g1_storage.high(), "Bad shrink!");

   _g1_committed.set_end(mr.start());

   _expansion_regions += num_regions_deleted;

   // Tell the cardtable about it.

   Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);

   // And the offset table as well.

   _bot_shared->resize(_g1_committed.word_size());

   HeapRegionRemSet::shrink_heap(n_regions());

   if (Verbose && PrintGC) {

     size_t new_mem_size = _g1_storage.committed_size();

     gclog_or_tty->print_cr("Shrinking garbage-first heap from %ldK by %ldK to %ldK",

                            old_mem_size/K, aligned_shrink_bytes/K,

                            new_mem_size/K);

   }

 }

 void G1CollectedHeap::shrink(size_t shrink_bytes) {

   verify_region_sets_optional();

   release_gc_alloc_regions(true /* totally */);

   // Instead of tearing down / rebuilding the free lists here, we

   // could instead use the remove_all_pending() method on free_list to

   // remove only the ones that we need to remove.

   tear_down_region_lists();  // We will rebuild them in a moment.

   shrink_helper(shrink_bytes);

   rebuild_region_lists();

   verify_region_sets_optional();

 }

 // Public methods.

 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away

 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list

 #endif // _MSC_VER

 G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) :

   SharedHeap(policy_),

   _g1_policy(policy_),

   _dirty_card_queue_set(false),

   _into_cset_dirty_card_queue_set(false),

   _is_alive_closure(this),

   _ref_processor(NULL),

   _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)),

   _bot_shared(NULL),

   _objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL),

   _evac_failure_scan_stack(NULL) ,

   _mark_in_progress(false),

   _cg1r(NULL), _summary_bytes_used(0),

   _cur_alloc_region(NULL),

   _refine_cte_cl(NULL),

   _full_collection(false),

   _free_list("Master Free List"),

   _secondary_free_list("Secondary Free List"),

   _humongous_set("Master Humongous Set"),

   _free_regions_coming(false),

   _young_list(new YoungList(this)),

   _gc_time_stamp(0),

   _surviving_young_words(NULL),

   _full_collections_completed(0),

   _in_cset_fast_test(NULL),

   _in_cset_fast_test_base(NULL),

   _dirty_cards_region_list(NULL) {

   _g1h = this; // To catch bugs.

   if (_process_strong_tasks == NULL || !_process_strong_tasks->valid()) {

     vm_exit_during_initialization("Failed necessary allocation.");

   }

   _humongous_object_threshold_in_words = HeapRegion::GrainWords / 2;

   int n_queues = MAX2((int)ParallelGCThreads, 1);

   _task_queues = new RefToScanQueueSet(n_queues);

   int n_rem_sets = HeapRegionRemSet::num_par_rem_sets();

   assert(n_rem_sets > 0, "Invariant.");

   HeapRegionRemSetIterator** iter_arr =

     NEW_C_HEAP_ARRAY(HeapRegionRemSetIterator*, n_queues);

   for (int i = 0; i < n_queues; i++) {

     iter_arr[i] = new HeapRegionRemSetIterator();

   }

   _rem_set_iterator = iter_arr;

   for (int i = 0; i < n_queues; i++) {

     RefToScanQueue* q = new RefToScanQueue();

     q->initialize();

     _task_queues->register_queue(i, q);

   }

   for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {

     _gc_alloc_regions[ap]          = NULL;

     _gc_alloc_region_counts[ap]    = 0;

     _retained_gc_alloc_regions[ap] = NULL;

     // by default, we do not retain a GC alloc region for each ap;

     // we'll override this, when appropriate, below

     _retain_gc_alloc_region[ap]    = false;

   }

   // We will try to remember the last half-full tenured region we

   // allocated to at the end of a collection so that we can re-use it

   // during the next collection.

   _retain_gc_alloc_region[GCAllocForTenured]  = true;

   guarantee(_task_queues != NULL, "task_queues allocation failure.");

 }

 jint G1CollectedHeap::initialize() {

   CollectedHeap::pre_initialize();

   os::enable_vtime();

   // Necessary to satisfy locking discipline assertions.

   MutexLocker x(Heap_lock);

   // While there are no constraints in the GC code that HeapWordSize

   // be any particular value, there are multiple other areas in the

   // system which believe this to be true (e.g. oop->object_size in some

   // cases incorrectly returns the size in wordSize units rather than

   // HeapWordSize).

   guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize");

   size_t init_byte_size = collector_policy()->initial_heap_byte_size();

   size_t max_byte_size = collector_policy()->max_heap_byte_size();

   // Ensure that the sizes are properly aligned.

   Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap");

   Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap");

   _cg1r = new ConcurrentG1Refine();

   // Reserve the maximum.

   PermanentGenerationSpec* pgs = collector_policy()->permanent_generation();

   // Includes the perm-gen.

   const size_t total_reserved = max_byte_size + pgs->max_size();

   char* addr = Universe::preferred_heap_base(total_reserved, Universe::UnscaledNarrowOop);

   ReservedSpace heap_rs(max_byte_size + pgs->max_size(),

                         HeapRegion::GrainBytes,

                         UseLargePages, addr);

   if (UseCompressedOops) {

     if (addr != NULL && !heap_rs.is_reserved()) {

       // Failed to reserve at specified address - the requested memory

       // region is taken already, for example, by 'java' launcher.

       // Try again to reserver heap higher.

       addr = Universe::preferred_heap_base(total_reserved, Universe::ZeroBasedNarrowOop);

       ReservedSpace heap_rs0(total_reserved, HeapRegion::GrainBytes,

                              UseLargePages, addr);

       if (addr != NULL && !heap_rs0.is_reserved()) {

         // Failed to reserve at specified address again - give up.

         addr = Universe::preferred_heap_base(total_reserved, Universe::HeapBasedNarrowOop);

         assert(addr == NULL, "");

         ReservedSpace heap_rs1(total_reserved, HeapRegion::GrainBytes,

                                UseLargePages, addr);

         heap_rs = heap_rs1;

       } else {

         heap_rs = heap_rs0;

       }

     }

   }

   if (!heap_rs.is_reserved()) {

     vm_exit_during_initialization("Could not reserve enough space for object heap");

     return JNI_ENOMEM;

   }

   // It is important to do this in a way such that concurrent readers can't

   // temporarily think somethings in the heap.  (I've actually seen this

   // happen in asserts: DLD.)

   _reserved.set_word_size(0);

   _reserved.set_start((HeapWord*)heap_rs.base());

   _reserved.set_end((HeapWord*)(heap_rs.base() + heap_rs.size()));

   _expansion_regions = max_byte_size/HeapRegion::GrainBytes;

   // Create the gen rem set (and barrier set) for the entire reserved region.

   _rem_set = collector_policy()->create_rem_set(_reserved, 2);

   set_barrier_set(rem_set()->bs());

   if (barrier_set()->is_a(BarrierSet::ModRef)) {

     _mr_bs = (ModRefBarrierSet*)_barrier_set;

   } else {

     vm_exit_during_initialization("G1 requires a mod ref bs.");

     return JNI_ENOMEM;

   }

   // Also create a G1 rem set.

   if (mr_bs()->is_a(BarrierSet::CardTableModRef)) {

     _g1_rem_set = new G1RemSet(this, (CardTableModRefBS*)mr_bs());

   } else {

     vm_exit_during_initialization("G1 requires a cardtable mod ref bs.");

     return JNI_ENOMEM;

   }

   // Carve out the G1 part of the heap.

   ReservedSpace g1_rs   = heap_rs.first_part(max_byte_size);

   _g1_reserved = MemRegion((HeapWord*)g1_rs.base(),

                            g1_rs.size()/HeapWordSize);

   ReservedSpace perm_gen_rs = heap_rs.last_part(max_byte_size);

   _perm_gen = pgs->init(perm_gen_rs, pgs->init_size(), rem_set());

   _g1_storage.initialize(g1_rs, 0);

   _g1_committed = MemRegion((HeapWord*)_g1_storage.low(), (size_t) 0);

   _g1_max_committed = _g1_committed;

   _hrs = new HeapRegionSeq(_expansion_regions);

   guarantee(_hrs != NULL, "Couldn't allocate HeapRegionSeq");

   guarantee(_cur_alloc_region == NULL, "from constructor");

   // 6843694 - ensure that the maximum region index can fit

   // in the remembered set structures.

   const size_t max_region_idx = ((size_t)1 << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1;

   guarantee((max_regions() - 1) <= max_region_idx, "too many regions");

   size_t max_cards_per_region = ((size_t)1 << (sizeof(CardIdx_t)*BitsPerByte-1)) - 1;

   guarantee(HeapRegion::CardsPerRegion > 0, "make sure it's initialized");

   guarantee((size_t) HeapRegion::CardsPerRegion < max_cards_per_region,

             "too many cards per region");

   HeapRegionSet::set_unrealistically_long_length(max_regions() + 1);

   _bot_shared = new G1BlockOffsetSharedArray(_reserved,

                                              heap_word_size(init_byte_size));

   _g1h = this;

    _in_cset_fast_test_length = max_regions();

    _in_cset_fast_test_base = NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);

    // We're biasing _in_cset_fast_test to avoid subtracting the

    // beginning of the heap every time we want to index; basically

    // it's the same with what we do with the card table.

    _in_cset_fast_test = _in_cset_fast_test_base -

                 ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);

    // Clear the _cset_fast_test bitmap in anticipation of adding

    // regions to the incremental collection set for the first

    // evacuation pause.

    clear_cset_fast_test();

   // Create the ConcurrentMark data structure and thread.

   // (Must do this late, so that "max_regions" is defined.)

   _cm       = new ConcurrentMark(heap_rs, (int) max_regions());

   _cmThread = _cm->cmThread();

   // Initialize the from_card cache structure of HeapRegionRemSet.

   HeapRegionRemSet::init_heap(max_regions());

   // Now expand into the initial heap size.

   if (!expand(init_byte_size)) {

     vm_exit_during_initialization("Failed to allocate initial heap.");

     return JNI_ENOMEM;

   }

   // Perform any initialization actions delegated to the policy.

   g1_policy()->init();

   g1_policy()->note_start_of_mark_thread();

   _refine_cte_cl =

     new RefineCardTableEntryClosure(ConcurrentG1RefineThread::sts(),

                                     g1_rem_set(),

                                     concurrent_g1_refine());

   JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

   JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,

                                                SATB_Q_FL_lock,

                                                G1SATBProcessCompletedThreshold,

                                                Shared_SATB_Q_lock);

   JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,

                                                 DirtyCardQ_FL_lock,

                                                 concurrent_g1_refine()->yellow_zone(),

                                                 concurrent_g1_refine()->red_zone(),

                                                 Shared_DirtyCardQ_lock);

   if (G1DeferredRSUpdate) {

     dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,

                                       DirtyCardQ_FL_lock,

                                       -1, // never trigger processing

                                       -1, // no limit on length

                                       Shared_DirtyCardQ_lock,

                                       &JavaThread::dirty_card_queue_set());

   }

   // Initialize the card queue set used to hold cards containing

   // references into the collection set.

   _into_cset_dirty_card_queue_set.initialize(DirtyCardQ_CBL_mon,

                                              DirtyCardQ_FL_lock,

                                              -1, // never trigger processing

                                              -1, // no limit on length

                                              Shared_DirtyCardQ_lock,

                                              &JavaThread::dirty_card_queue_set());

   // In case we're keeping closure specialization stats, initialize those

   // counts and that mechanism.

   SpecializationStats::clear();

   _gc_alloc_region_list = NULL;

   // Do later initialization work for concurrent refinement.

   _cg1r->init();

   return JNI_OK;

 }

 void G1CollectedHeap::ref_processing_init() {

   // Reference processing in G1 currently works as follows:

   //

   // * There is only one reference processor instance that

   //   'spans' the entire heap. It is created by the code

   //   below.

   // * Reference discovery is not enabled during an incremental

   //   pause (see 6484982).

   // * Discoverered refs are not enqueued nor are they processed

   //   during an incremental pause (see 6484982).

   // * Reference discovery is enabled at initial marking.

   // * Reference discovery is disabled and the discovered

   //   references processed etc during remarking.

   // * Reference discovery is MT (see below).

   // * Reference discovery requires a barrier (see below).

   // * Reference processing is currently not MT (see 6608385).

   // * A full GC enables (non-MT) reference discovery and

   //   processes any discovered references.

   SharedHeap::ref_processing_init();

   MemRegion mr = reserved_region();

   _ref_processor = ReferenceProcessor::create_ref_processor(

                                          mr,    // span

                                          false, // Reference discovery is not atomic

                                          true,  // mt_discovery

                                          &_is_alive_closure, // is alive closure

                                                              // for efficiency

                                          ParallelGCThreads,

                                          ParallelRefProcEnabled,

                                          true); // Setting next fields of discovered

                                                 // lists requires a barrier.

 }

 size_t G1CollectedHeap::capacity() const {

   return _g1_committed.byte_size();

 }

 void G1CollectedHeap::iterate_dirty_card_closure(CardTableEntryClosure* cl,

                                                  DirtyCardQueue* into_cset_dcq,

                                                  bool concurrent,

                                                  int worker_i) {

   // Clean cards in the hot card cache

   concurrent_g1_refine()->clean_up_cache(worker_i, g1_rem_set(), into_cset_dcq);

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   int n_completed_buffers = 0;

   while (dcqs.apply_closure_to_completed_buffer(cl, worker_i, 0, true)) {

     n_completed_buffers++;

   }

   g1_policy()->record_update_rs_processed_buffers(worker_i,

                                                   (double) n_completed_buffers);

   dcqs.clear_n_completed_buffers();

   assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!");

 }

 // Computes the sum of the storage used by the various regions.

 size_t G1CollectedHeap::used() const {

   assert(Heap_lock->owner() != NULL,

          "Should be owned on this thread's behalf.");

   size_t result = _summary_bytes_used;

   // Read only once in case it is set to NULL concurrently

   HeapRegion* hr = _cur_alloc_region;

   if (hr != NULL)

     result += hr->used();

   return result;

 }

 size_t G1CollectedHeap::used_unlocked() const {

   size_t result = _summary_bytes_used;

   return result;

 }

 class SumUsedClosure: public HeapRegionClosure {

   size_t _used;

 public:

   SumUsedClosure() : _used(0) {}

   bool doHeapRegion(HeapRegion* r) {

     if (!r->continuesHumongous()) {

       _used += r->used();

     }

     return false;

   }

   size_t result() { return _used; }

 };

 size_t G1CollectedHeap::recalculate_used() const {

   SumUsedClosure blk;

   _hrs->iterate(&blk);

   return blk.result();

 }

 #ifndef PRODUCT

 class SumUsedRegionsClosure: public HeapRegionClosure {

   size_t _num;

 public:

   SumUsedRegionsClosure() : _num(0) {}

   bool doHeapRegion(HeapRegion* r) {

     if (r->continuesHumongous() || r->used() > 0 || r->is_gc_alloc_region()) {

       _num += 1;

     }

     return false;

   }

   size_t result() { return _num; }

 };

 size_t G1CollectedHeap::recalculate_used_regions() const {

   SumUsedRegionsClosure blk;

   _hrs->iterate(&blk);

   return blk.result();

 }

 #endif // PRODUCT

 size_t G1CollectedHeap::unsafe_max_alloc() {

   if (free_regions() > 0) return HeapRegion::GrainBytes;

   // otherwise, is there space in the current allocation region?

   // We need to store the current allocation region in a local variable

   // here. The problem is that this method doesn't take any locks and

   // there may be other threads which overwrite the current allocation

   // region field. attempt_allocation(), for example, sets it to NULL

   // and this can happen *after* the NULL check here but before the call

   // to free(), resulting in a SIGSEGV. Note that this doesn't appear

   // to be a problem in the optimized build, since the two loads of the

   // current allocation region field are optimized away.

   HeapRegion* car = _cur_alloc_region;

   // FIXME: should iterate over all regions?

   if (car == NULL) {

     return 0;

   }

   return car->free();

 }

 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {

   return

     ((cause == GCCause::_gc_locker           && GCLockerInvokesConcurrent) ||

      (cause == GCCause::_java_lang_system_gc && ExplicitGCInvokesConcurrent));

 }

 void G1CollectedHeap::increment_full_collections_completed(bool concurrent) {

   MonitorLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag);

   // We assume that if concurrent == true, then the caller is a

   // concurrent thread that was joined the Suspendible Thread

   // Set. If there's ever a cheap way to check this, we should add an

   // assert here.

   // We have already incremented _total_full_collections at the start

   // of the GC, so total_full_collections() represents how many full

   // collections have been started.

   unsigned int full_collections_started = total_full_collections();

   // Given that this method is called at the end of a Full GC or of a

   // concurrent cycle, and those can be nested (i.e., a Full GC can

   // interrupt a concurrent cycle), the number of full collections

   // completed should be either one (in the case where there was no

   // nesting) or two (when a Full GC interrupted a concurrent cycle)

   // behind the number of full collections started.

   // This is the case for the inner caller, i.e. a Full GC.

   assert(concurrent ||

          (full_collections_started == _full_collections_completed + 1) ||

          (full_collections_started == _full_collections_completed + 2),

          err_msg("for inner caller (Full GC): full_collections_started = %u "

                  "is inconsistent with _full_collections_completed = %u",

                  full_collections_started, _full_collections_completed));

   // This is the case for the outer caller, i.e. the concurrent cycle.

   assert(!concurrent ||

          (full_collections_started == _full_collections_completed + 1),

          err_msg("for outer caller (concurrent cycle): "

                  "full_collections_started = %u "

                  "is inconsistent with _full_collections_completed = %u",

                  full_collections_started, _full_collections_completed));

   _full_collections_completed += 1;

   // We need to clear the "in_progress" flag in the CM thread before

   // we wake up any waiters (especially when ExplicitInvokesConcurrent

   // is set) so that if a waiter requests another System.gc() it doesn't

   // incorrectly see that a marking cyle is still in progress.

   if (concurrent) {

     _cmThread->clear_in_progress();

   }

   // This notify_all() will ensure that a thread that called

   // System.gc() with (with ExplicitGCInvokesConcurrent set or not)

   // and it's waiting for a full GC to finish will be woken up. It is

   // waiting in VM_G1IncCollectionPause::doit_epilogue().

   FullGCCount_lock->notify_all();

 }

 void G1CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) {

   assert_at_safepoint(true /* should_be_vm_thread */);

   GCCauseSetter gcs(this, cause);

   switch (cause) {

     case GCCause::_heap_inspection:

     case GCCause::_heap_dump: {

       HandleMark hm;

       do_full_collection(false);         // don't clear all soft refs

       break;

     }

     default: // XXX FIX ME

       ShouldNotReachHere(); // Unexpected use of this function

   }

 }

 void G1CollectedHeap::collect(GCCause::Cause cause) {

   // The caller doesn't have the Heap_lock

   assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock");

   unsigned int gc_count_before;

   unsigned int full_gc_count_before;

   {

     MutexLocker ml(Heap_lock);

     // Read the GC count while holding the Heap_lock

     gc_count_before = SharedHeap::heap()->total_collections();

     full_gc_count_before = SharedHeap::heap()->total_full_collections();

   }

   if (should_do_concurrent_full_gc(cause)) {

     // Schedule an initial-mark evacuation pause that will start a

     // concurrent cycle. We're setting word_size to 0 which means that

     // we are not requesting a post-GC allocation.

     VM_G1IncCollectionPause op(gc_count_before,

                                0,     /* word_size */

                                true,  /* should_initiate_conc_mark */

                                g1_policy()->max_pause_time_ms(),

                                cause);

     VMThread::execute(&op);

   } else {

     if (cause == GCCause::_gc_locker

         DEBUG_ONLY(|| cause == GCCause::_scavenge_alot)) {

       // Schedule a standard evacuation pause. We're setting word_size

       // to 0 which means that we are not requesting a post-GC allocation.

       VM_G1IncCollectionPause op(gc_count_before,

                                  0,     /* word_size */

                                  false, /* should_initiate_conc_mark */

                                  g1_policy()->max_pause_time_ms(),

                                  cause);

       VMThread::execute(&op);

     } else {

       // Schedule a Full GC.

       VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);

       VMThread::execute(&op);

     }

   }

 }

 bool G1CollectedHeap::is_in(const void* p) const {

   if (_g1_committed.contains(p)) {

     HeapRegion* hr = _hrs->addr_to_region(p);

     return hr->is_in(p);

   } else {

     return _perm_gen->as_gen()->is_in(p);

   }

 }

 // Iteration functions.

 // Iterates an OopClosure over all ref-containing fields of objects

 // within a HeapRegion.

 class IterateOopClosureRegionClosure: public HeapRegionClosure {

   MemRegion _mr;

   OopClosure* _cl;

 public:

   IterateOopClosureRegionClosure(MemRegion mr, OopClosure* cl)

     : _mr(mr), _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     if (! r->continuesHumongous()) {

       r->oop_iterate(_cl);

     }

     return false;

   }

 };

 void G1CollectedHeap::oop_iterate(OopClosure* cl, bool do_perm) {

   IterateOopClosureRegionClosure blk(_g1_committed, cl);

   _hrs->iterate(&blk);

   if (do_perm) {

     perm_gen()->oop_iterate(cl);

   }

 }

 void G1CollectedHeap::oop_iterate(MemRegion mr, OopClosure* cl, bool do_perm) {

   IterateOopClosureRegionClosure blk(mr, cl);

   _hrs->iterate(&blk);

   if (do_perm) {

     perm_gen()->oop_iterate(cl);

   }

 }

 // Iterates an ObjectClosure over all objects within a HeapRegion.

 class IterateObjectClosureRegionClosure: public HeapRegionClosure {

   ObjectClosure* _cl;

 public:

   IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     if (! r->continuesHumongous()) {

       r->object_iterate(_cl);

     }

     return false;

   }

 };

 void G1CollectedHeap::object_iterate(ObjectClosure* cl, bool do_perm) {

   IterateObjectClosureRegionClosure blk(cl);

   _hrs->iterate(&blk);

   if (do_perm) {

     perm_gen()->object_iterate(cl);

   }

 }

 void G1CollectedHeap::object_iterate_since_last_GC(ObjectClosure* cl) {

   // FIXME: is this right?

   guarantee(false, "object_iterate_since_last_GC not supported by G1 heap");

 }

 // Calls a SpaceClosure on a HeapRegion.

 class SpaceClosureRegionClosure: public HeapRegionClosure {

   SpaceClosure* _cl;

 public:

   SpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     _cl->do_space(r);

     return false;

   }

 };

 void G1CollectedHeap::space_iterate(SpaceClosure* cl) {

   SpaceClosureRegionClosure blk(cl);

   _hrs->iterate(&blk);

 }

 void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) {

   _hrs->iterate(cl);

 }

 void G1CollectedHeap::heap_region_iterate_from(HeapRegion* r,

                                                HeapRegionClosure* cl) {

   _hrs->iterate_from(r, cl);

 }

 void

 G1CollectedHeap::heap_region_iterate_from(int idx, HeapRegionClosure* cl) {

   _hrs->iterate_from(idx, cl);

 }

 HeapRegion* G1CollectedHeap::region_at(size_t idx) { return _hrs->at(idx); }

 void

 G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl,

                                                  int worker,

                                                  jint claim_value) {

   const size_t regions = n_regions();

   const size_t worker_num = (G1CollectedHeap::use_parallel_gc_threads() ? ParallelGCThreads : 1);

   // try to spread out the starting points of the workers

   const size_t start_index = regions / worker_num * (size_t) worker;

   // each worker will actually look at all regions

   for (size_t count = 0; count < regions; ++count) {

     const size_t index = (start_index + count) % regions;

     assert(0 <= index && index < regions, "sanity");

     HeapRegion* r = region_at(index);

     // we'll ignore "continues humongous" regions (we'll process them

     // when we come across their corresponding "start humongous"

     // region) and regions already claimed

     if (r->claim_value() == claim_value || r->continuesHumongous()) {

       continue;

     }

     // OK, try to claim it

     if (r->claimHeapRegion(claim_value)) {

       // success!

       assert(!r->continuesHumongous(), "sanity");

       if (r->startsHumongous()) {

         // If the region is "starts humongous" we'll iterate over its

         // "continues humongous" first; in fact we'll do them

         // first. The order is important. In on case, calling the

         // closure on the "starts humongous" region might de-allocate

         // and clear all its "continues humongous" regions and, as a

         // result, we might end up processing them twice. So, we'll do

         // them first (notice: most closures will ignore them anyway) and

         // then we'll do the "starts humongous" region.

         for (size_t ch_index = index + 1; ch_index < regions; ++ch_index) {

           HeapRegion* chr = region_at(ch_index);

           // if the region has already been claimed or it's not

           // "continues humongous" we're done

           if (chr->claim_value() == claim_value ||

               !chr->continuesHumongous()) {

             break;

           }

           // Noone should have claimed it directly. We can given

           // that we claimed its "starts humongous" region.

           assert(chr->claim_value() != claim_value, "sanity");

           assert(chr->humongous_start_region() == r, "sanity");

           if (chr->claimHeapRegion(claim_value)) {

             // we should always be able to claim it; noone else should

             // be trying to claim this region

             bool res2 = cl->doHeapRegion(chr);

             assert(!res2, "Should not abort");

             // Right now, this holds (i.e., no closure that actually

             // does something with "continues humongous" regions

             // clears them). We might have to weaken it in the future,

             // but let's leave these two asserts here for extra safety.

             assert(chr->continuesHumongous(), "should still be the case");

             assert(chr->humongous_start_region() == r, "sanity");

           } else {

             guarantee(false, "we should not reach here");

           }

         }

       }

       assert(!r->continuesHumongous(), "sanity");

       bool res = cl->doHeapRegion(r);

       assert(!res, "Should not abort");

     }

   }

 }

 class ResetClaimValuesClosure: public HeapRegionClosure {

 public:

   bool doHeapRegion(HeapRegion* r) {

     r->set_claim_value(HeapRegion::InitialClaimValue);

     return false;

   }

 };

 void

 G1CollectedHeap::reset_heap_region_claim_values() {

   ResetClaimValuesClosure blk;

   heap_region_iterate(&blk);

 }

 #ifdef ASSERT

 // This checks whether all regions in the heap have the correct claim

 // value. I also piggy-backed on this a check to ensure that the

 // humongous_start_region() information on "continues humongous"

 // regions is correct.

 class CheckClaimValuesClosure : public HeapRegionClosure {

 private:

   jint _claim_value;

   size_t _failures;

   HeapRegion* _sh_region;

 public:

   CheckClaimValuesClosure(jint claim_value) :

     _claim_value(claim_value), _failures(0), _sh_region(NULL) { }

   bool doHeapRegion(HeapRegion* r) {

     if (r->claim_value() != _claim_value) {

       gclog_or_tty->print_cr("Region ["PTR_FORMAT","PTR_FORMAT"), "