Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "precompiled.hpp"

 #include "code/icBuffer.hpp"

 #include "gc_implementation/g1/bufferingOopClosure.hpp"

 #include "gc_implementation/g1/concurrentG1Refine.hpp"

 #include "gc_implementation/g1/concurrentG1RefineThread.hpp"

 #include "gc_implementation/g1/concurrentMarkThread.inline.hpp"

 #include "gc_implementation/g1/concurrentZFThread.hpp"

 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

 #include "gc_implementation/g1/g1CollectorPolicy.hpp"

 #include "gc_implementation/g1/g1MarkSweep.hpp"

 #include "gc_implementation/g1/g1OopClosures.inline.hpp"

 #include "gc_implementation/g1/g1RemSet.inline.hpp"

 #include "gc_implementation/g1/heapRegionRemSet.hpp"

 #include "gc_implementation/g1/heapRegionSeq.inline.hpp"

 #include "gc_implementation/g1/vm_operations_g1.hpp"

 #include "gc_implementation/shared/isGCActiveMark.hpp"

 #include "memory/gcLocker.inline.hpp"

 #include "memory/genOopClosures.inline.hpp"

 #include "memory/generationSpec.hpp"

 #include "oops/oop.inline.hpp"

 #include "oops/oop.pcgc.inline.hpp"

 #include "runtime/aprofiler.hpp"

 #include "runtime/vmThread.hpp"

 size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0;

 // turn it on so that the contents of the young list (scan-only /

 // to-be-collected) are printed at "strategic" points before / during

 // / after the collection --- this is useful for debugging

 #define YOUNG_LIST_VERBOSE 0

 // CURRENT STATUS

 // This file is under construction.  Search for "FIXME".

 // INVARIANTS/NOTES

 //

 // All allocation activity covered by the G1CollectedHeap interface is

 // serialized by acquiring the HeapLock.  This happens in mem_allocate

 // and allocate_new_tlab, which are the "entry" points to the

 // allocation code from the rest of the JVM.  (Note that this does not

 // apply to TLAB allocation, which is not part of this interface: it

 // is done by clients of this interface.)

 // Local to this file.

 class RefineCardTableEntryClosure: public CardTableEntryClosure {

   SuspendibleThreadSet* _sts;

   G1RemSet* _g1rs;

   ConcurrentG1Refine* _cg1r;

   bool _concurrent;

 public:

   RefineCardTableEntryClosure(SuspendibleThreadSet* sts,

                               G1RemSet* g1rs,

                               ConcurrentG1Refine* cg1r) :

     _sts(sts), _g1rs(g1rs), _cg1r(cg1r), _concurrent(true)

   {}

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     bool oops_into_cset = _g1rs->concurrentRefineOneCard(card_ptr, worker_i, false);

     // This path is executed by the concurrent refine or mutator threads,

     // concurrently, and so we do not care if card_ptr contains references

     // that point into the collection set.

     assert(!oops_into_cset, "should be");

     if (_concurrent && _sts->should_yield()) {

       // Caller will actually yield.

       return false;

     }

     // Otherwise, we finished successfully; return true.

     return true;

   }

   void set_concurrent(bool b) { _concurrent = b; }

 };

 class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure {

   int _calls;

   G1CollectedHeap* _g1h;

   CardTableModRefBS* _ctbs;

   int _histo[256];

 public:

   ClearLoggedCardTableEntryClosure() :

     _calls(0)

   {

     _g1h = G1CollectedHeap::heap();

     _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

     for (int i = 0; i < 256; i++) _histo[i] = 0;

   }

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

       _calls++;

       unsigned char* ujb = (unsigned char*)card_ptr;

       int ind = (int)(*ujb);

       _histo[ind]++;

       *card_ptr = -1;

     }

     return true;

   }

   int calls() { return _calls; }

   void print_histo() {

     gclog_or_tty->print_cr("Card table value histogram:");

     for (int i = 0; i < 256; i++) {

       if (_histo[i] != 0) {

         gclog_or_tty->print_cr("  %d: %d", i, _histo[i]);

       }

     }

   }

 };

 class RedirtyLoggedCardTableEntryClosure: public CardTableEntryClosure {

   int _calls;

   G1CollectedHeap* _g1h;

   CardTableModRefBS* _ctbs;

 public:

   RedirtyLoggedCardTableEntryClosure() :

     _calls(0)

   {

     _g1h = G1CollectedHeap::heap();

     _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

   }

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

       _calls++;

       *card_ptr = 0;

     }

     return true;

   }

   int calls() { return _calls; }

 };

 class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure {

 public:

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     *card_ptr = CardTableModRefBS::dirty_card_val();

     return true;

   }

 };

 YoungList::YoungList(G1CollectedHeap* g1h)

   : _g1h(g1h), _head(NULL),

     _length(0),

     _last_sampled_rs_lengths(0),

     _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0)

 {

   guarantee( check_list_empty(false), "just making sure..." );

 }

 void YoungList::push_region(HeapRegion *hr) {

   assert(!hr->is_young(), "should not already be young");

   assert(hr->get_next_young_region() == NULL, "cause it should!");

   hr->set_next_young_region(_head);

   _head = hr;

   hr->set_young();

   double yg_surv_rate = _g1h->g1_policy()->predict_yg_surv_rate((int)_length);

   ++_length;

 }

 void YoungList::add_survivor_region(HeapRegion* hr) {

   assert(hr->is_survivor(), "should be flagged as survivor region");

   assert(hr->get_next_young_region() == NULL, "cause it should!");

   hr->set_next_young_region(_survivor_head);

   if (_survivor_head == NULL) {

     _survivor_tail = hr;

   }

   _survivor_head = hr;

   ++_survivor_length;

 }

 void YoungList::empty_list(HeapRegion* list) {

   while (list != NULL) {

     HeapRegion* next = list->get_next_young_region();

     list->set_next_young_region(NULL);

     list->uninstall_surv_rate_group();

     list->set_not_young();

     list = next;

   }

 }

 void YoungList::empty_list() {

   assert(check_list_well_formed(), "young list should be well formed");

   empty_list(_head);

   _head = NULL;

   _length = 0;

   empty_list(_survivor_head);

   _survivor_head = NULL;

   _survivor_tail = NULL;

   _survivor_length = 0;

   _last_sampled_rs_lengths = 0;

   assert(check_list_empty(false), "just making sure...");

 }

 bool YoungList::check_list_well_formed() {

   bool ret = true;

   size_t length = 0;

   HeapRegion* curr = _head;

   HeapRegion* last = NULL;

   while (curr != NULL) {

     if (!curr->is_young()) {

       gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" "

                              "incorrectly tagged (y: %d, surv: %d)",

                              curr->bottom(), curr->end(),

                              curr->is_young(), curr->is_survivor());

       ret = false;

     }

     ++length;

     last = curr;

     curr = curr->get_next_young_region();

   }

   ret = ret && (length == _length);

   if (!ret) {

     gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!");

     gclog_or_tty->print_cr("###   list has %d entries, _length is %d",

                            length, _length);

   }

   return ret;

 }

 bool YoungList::check_list_empty(bool check_sample) {

   bool ret = true;

   if (_length != 0) {

     gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %d",

                   _length);

     ret = false;

   }

   if (check_sample && _last_sampled_rs_lengths != 0) {

     gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths");

     ret = false;

   }

   if (_head != NULL) {

     gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head");

     ret = false;

   }

   if (!ret) {

     gclog_or_tty->print_cr("### YOUNG LIST does not seem empty");

   }

   return ret;

 }

 void

 YoungList::rs_length_sampling_init() {

   _sampled_rs_lengths = 0;

   _curr               = _head;

 }

 bool

 YoungList::rs_length_sampling_more() {

   return _curr != NULL;

 }

 void

 YoungList::rs_length_sampling_next() {

   assert( _curr != NULL, "invariant" );

   size_t rs_length = _curr->rem_set()->occupied();

   _sampled_rs_lengths += rs_length;

   // The current region may not yet have been added to the

   // incremental collection set (it gets added when it is

   // retired as the current allocation region).

   if (_curr->in_collection_set()) {

     // Update the collection set policy information for this region

     _g1h->g1_policy()->update_incremental_cset_info(_curr, rs_length);

   }

   _curr = _curr->get_next_young_region();

   if (_curr == NULL) {

     _last_sampled_rs_lengths = _sampled_rs_lengths;

     // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths);

   }

 }

 void

 YoungList::reset_auxilary_lists() {

   guarantee( is_empty(), "young list should be empty" );

   assert(check_list_well_formed(), "young list should be well formed");

   // Add survivor regions to SurvRateGroup.

   _g1h->g1_policy()->note_start_adding_survivor_regions();

   _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */);

   for (HeapRegion* curr = _survivor_head;

        curr != NULL;

        curr = curr->get_next_young_region()) {

     _g1h->g1_policy()->set_region_survivors(curr);

     // The region is a non-empty survivor so let's add it to

     // the incremental collection set for the next evacuation

     // pause.

     _g1h->g1_policy()->add_region_to_incremental_cset_rhs(curr);

   }

   _g1h->g1_policy()->note_stop_adding_survivor_regions();

   _head   = _survivor_head;

   _length = _survivor_length;

   if (_survivor_head != NULL) {

     assert(_survivor_tail != NULL, "cause it shouldn't be");

     assert(_survivor_length > 0, "invariant");

     _survivor_tail->set_next_young_region(NULL);

   }

   // Don't clear the survivor list handles until the start of

   // the next evacuation pause - we need it in order to re-tag

   // the survivor regions from this evacuation pause as 'young'

   // at the start of the next.

   _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */);

   assert(check_list_well_formed(), "young list should be well formed");

 }

 void YoungList::print() {

   HeapRegion* lists[] = {_head,   _survivor_head};

   const char* names[] = {"YOUNG", "SURVIVOR"};

   for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) {

     gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]);

     HeapRegion *curr = lists[list];

     if (curr == NULL)

       gclog_or_tty->print_cr("  empty");

     while (curr != NULL) {

       gclog_or_tty->print_cr("  [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, "

                              "age: %4d, y: %d, surv: %d",

                              curr->bottom(), curr->end(),

                              curr->top(),

                              curr->prev_top_at_mark_start(),

                              curr->next_top_at_mark_start(),

                              curr->top_at_conc_mark_count(),

                              curr->age_in_surv_rate_group_cond(),

                              curr->is_young(),

                              curr->is_survivor());

       curr = curr->get_next_young_region();

     }

   }

   gclog_or_tty->print_cr("");

 }

 void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr)

 {

   // Claim the right to put the region on the dirty cards region list

   // by installing a self pointer.

   HeapRegion* next = hr->get_next_dirty_cards_region();

   if (next == NULL) {

     HeapRegion* res = (HeapRegion*)

       Atomic::cmpxchg_ptr(hr, hr->next_dirty_cards_region_addr(),

                           NULL);

     if (res == NULL) {

       HeapRegion* head;

       do {

         // Put the region to the dirty cards region list.

         head = _dirty_cards_region_list;

         next = (HeapRegion*)

           Atomic::cmpxchg_ptr(hr, &_dirty_cards_region_list, head);

         if (next == head) {

           assert(hr->get_next_dirty_cards_region() == hr,

                  "hr->get_next_dirty_cards_region() != hr");

           if (next == NULL) {

             // The last region in the list points to itself.

             hr->set_next_dirty_cards_region(hr);

           } else {

             hr->set_next_dirty_cards_region(next);

           }

         }

       } while (next != head);

     }

   }

 }

 HeapRegion* G1CollectedHeap::pop_dirty_cards_region()

 {

   HeapRegion* head;

   HeapRegion* hr;

   do {

     head = _dirty_cards_region_list;

     if (head == NULL) {

       return NULL;

     }

     HeapRegion* new_head = head->get_next_dirty_cards_region();

     if (head == new_head) {

       // The last region.

       new_head = NULL;

     }

     hr = (HeapRegion*)Atomic::cmpxchg_ptr(new_head, &_dirty_cards_region_list,

                                           head);

   } while (hr != head);

   assert(hr != NULL, "invariant");

   hr->set_next_dirty_cards_region(NULL);

   return hr;

 }

 void G1CollectedHeap::stop_conc_gc_threads() {

   _cg1r->stop();

   _czft->stop();

   _cmThread->stop();

 }

 void G1CollectedHeap::check_ct_logs_at_safepoint() {

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set();

   // Count the dirty cards at the start.

   CountNonCleanMemRegionClosure count1(this);

   ct_bs->mod_card_iterate(&count1);

   int orig_count = count1.n();

   // First clear the logged cards.

   ClearLoggedCardTableEntryClosure clear;

   dcqs.set_closure(&clear);

   dcqs.apply_closure_to_all_completed_buffers();

   dcqs.iterate_closure_all_threads(false);

   clear.print_histo();

   // Now ensure that there's no dirty cards.

   CountNonCleanMemRegionClosure count2(this);

   ct_bs->mod_card_iterate(&count2);

   if (count2.n() != 0) {

     gclog_or_tty->print_cr("Card table has %d entries; %d originally",

                            count2.n(), orig_count);

   }

   guarantee(count2.n() == 0, "Card table should be clean.");

   RedirtyLoggedCardTableEntryClosure redirty;

   JavaThread::dirty_card_queue_set().set_closure(&redirty);

   dcqs.apply_closure_to_all_completed_buffers();

   dcqs.iterate_closure_all_threads(false);

   gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.",

                          clear.calls(), orig_count);

   guarantee(redirty.calls() == clear.calls(),

             "Or else mechanism is broken.");

   CountNonCleanMemRegionClosure count3(this);

   ct_bs->mod_card_iterate(&count3);

   if (count3.n() != orig_count) {

     gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.",

                            orig_count, count3.n());

     guarantee(count3.n() >= orig_count, "Should have restored them all.");

   }

   JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

 }

 // Private class members.

 G1CollectedHeap* G1CollectedHeap::_g1h;

 // Private methods.

 // Finds a HeapRegion that can be used to allocate a given size of block.

 HeapRegion* G1CollectedHeap::newAllocRegion_work(size_t word_size,

                                                  bool do_expand,

                                                  bool zero_filled) {

   ConcurrentZFThread::note_region_alloc();

   HeapRegion* res = alloc_free_region_from_lists(zero_filled);

   if (res == NULL && do_expand) {

     expand(word_size * HeapWordSize);

     res = alloc_free_region_from_lists(zero_filled);

     assert(res == NULL ||

            (!res->isHumongous() &&

             (!zero_filled ||

              res->zero_fill_state() == HeapRegion::Allocated)),

            "Alloc Regions must be zero filled (and non-H)");

   }

   if (res != NULL) {

     if (res->is_empty()) {

       _free_regions--;

     }

     assert(!res->isHumongous() &&

            (!zero_filled || res->zero_fill_state() == HeapRegion::Allocated),

            err_msg("Non-young alloc Regions must be zero filled (and non-H):"

                    " res->isHumongous()=%d, zero_filled=%d, res->zero_fill_state()=%d",

                    res->isHumongous(), zero_filled, res->zero_fill_state()));

     assert(!res->is_on_unclean_list(),

            "Alloc Regions must not be on the unclean list");

     if (G1PrintHeapRegions) {

       gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT", "PTR_FORMAT"], "

                              "top "PTR_FORMAT,

                              res->hrs_index(), res->bottom(), res->end(), res->top());

     }

   }

   return res;

 }

 HeapRegion* G1CollectedHeap::newAllocRegionWithExpansion(int purpose,

                                                          size_t word_size,

                                                          bool zero_filled) {

   HeapRegion* alloc_region = NULL;

   if (_gc_alloc_region_counts[purpose] < g1_policy()->max_regions(purpose)) {

     alloc_region = newAllocRegion_work(word_size, true, zero_filled);

     if (purpose == GCAllocForSurvived && alloc_region != NULL) {

       alloc_region->set_survivor();

     }

     ++_gc_alloc_region_counts[purpose];

   } else {

     g1_policy()->note_alloc_region_limit_reached(purpose);

   }

   return alloc_region;

 }

 // If could fit into free regions w/o expansion, try.

 // Otherwise, if can expand, do so.

 // Otherwise, if using ex regions might help, try with ex given back.

 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size) {

   assert_heap_locked_or_at_safepoint();

   assert(regions_accounted_for(), "Region leakage!");

   // We can't allocate humongous regions while cleanupComplete is

   // running, since some of the regions we find to be empty might not

   // yet be added to the unclean list. If we're already at a

   // safepoint, this call is unnecessary, not to mention wrong.

   if (!SafepointSynchronize::is_at_safepoint()) {

     wait_for_cleanup_complete();

   }

   size_t num_regions =

          round_to(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords;

   // Special case if < one region???

   // Remember the ft size.

   size_t x_size = expansion_regions();

   HeapWord* res = NULL;

   bool eliminated_allocated_from_lists = false;

   // Can the allocation potentially fit in the free regions?

   if (free_regions() >= num_regions) {

     res = _hrs->obj_allocate(word_size);

   }

   if (res == NULL) {

     // Try expansion.

     size_t fs = _hrs->free_suffix();

     if (fs + x_size >= num_regions) {

       expand((num_regions - fs) * HeapRegion::GrainBytes);

       res = _hrs->obj_allocate(word_size);

       assert(res != NULL, "This should have worked.");

     } else {

       // Expansion won't help.  Are there enough free regions if we get rid

       // of reservations?

       size_t avail = free_regions();

       if (avail >= num_regions) {

         res = _hrs->obj_allocate(word_size);

         if (res != NULL) {

           remove_allocated_regions_from_lists();

           eliminated_allocated_from_lists = true;

         }

       }

     }

   }

   if (res != NULL) {

     // Increment by the number of regions allocated.

     // FIXME: Assumes regions all of size GrainBytes.

 #ifndef PRODUCT

     mr_bs()->verify_clean_region(MemRegion(res, res + num_regions *

                                            HeapRegion::GrainWords));

 #endif

     if (!eliminated_allocated_from_lists)

       remove_allocated_regions_from_lists();

     _summary_bytes_used += word_size * HeapWordSize;

     _free_regions -= num_regions;

     _num_humongous_regions += (int) num_regions;

   }

   assert(regions_accounted_for(), "Region Leakage");

   return res;

 }

 void

 G1CollectedHeap::retire_cur_alloc_region(HeapRegion* cur_alloc_region) {

   // The cleanup operation might update _summary_bytes_used

   // concurrently with this method. So, right now, if we don't wait

   // for it to complete, updates to _summary_bytes_used might get

   // lost. This will be resolved in the near future when the operation

   // of the free region list is revamped as part of CR 6977804.

   wait_for_cleanup_complete();

   retire_cur_alloc_region_common(cur_alloc_region);

   assert(_cur_alloc_region == NULL, "post-condition");

 }

 // See the comment in the .hpp file about the locking protocol and

 // assumptions of this method (and other related ones).

 HeapWord*

 G1CollectedHeap::replace_cur_alloc_region_and_allocate(size_t word_size,

                                                        bool at_safepoint,

                                                        bool do_dirtying,

                                                        bool can_expand) {

   assert_heap_locked_or_at_safepoint();

   assert(_cur_alloc_region == NULL,

          "replace_cur_alloc_region_and_allocate() should only be called "

          "after retiring the previous current alloc region");

   assert(SafepointSynchronize::is_at_safepoint() == at_safepoint,

          "at_safepoint and is_at_safepoint() should be a tautology");

   assert(!can_expand || g1_policy()->can_expand_young_list(),

          "we should not call this method with can_expand == true if "

          "we are not allowed to expand the young gen");

   if (can_expand || !g1_policy()->is_young_list_full()) {

     if (!at_safepoint) {

       // The cleanup operation might update _summary_bytes_used

       // concurrently with this method. So, right now, if we don't

       // wait for it to complete, updates to _summary_bytes_used might

       // get lost. This will be resolved in the near future when the

       // operation of the free region list is revamped as part of

       // CR 6977804. If we're already at a safepoint, this call is

       // unnecessary, not to mention wrong.

       wait_for_cleanup_complete();

     }

     HeapRegion* new_cur_alloc_region = newAllocRegion(word_size,

                                                       false /* zero_filled */);

     if (new_cur_alloc_region != NULL) {

       assert(new_cur_alloc_region->is_empty(),

              "the newly-allocated region should be empty, "

              "as right now we only allocate new regions out of the free list");

       g1_policy()->update_region_num(true /* next_is_young */);

       _summary_bytes_used -= new_cur_alloc_region->used();

       set_region_short_lived_locked(new_cur_alloc_region);

       assert(!new_cur_alloc_region->isHumongous(),

              "Catch a regression of this bug.");

       // We need to ensure that the stores to _cur_alloc_region and,

       // subsequently, to top do not float above the setting of the

       // young type.

       OrderAccess::storestore();

       // Now allocate out of the new current alloc region. We could

       // have re-used allocate_from_cur_alloc_region() but its

       // operation is slightly different to what we need here. First,

       // allocate_from_cur_alloc_region() is only called outside a

       // safepoint and will always unlock the Heap_lock if it returns

       // a non-NULL result. Second, it assumes that the current alloc

       // region is what's already assigned in _cur_alloc_region. What

       // we want here is to actually do the allocation first before we

       // assign the new region to _cur_alloc_region. This ordering is

       // not currently important, but it will be essential when we

       // change the code to support CAS allocation in the future (see

       // CR 6994297).

       //

       // This allocate method does BOT updates and we don't need them in

       // the young generation. This will be fixed in the near future by

       // CR 6994297.

       HeapWord* result = new_cur_alloc_region->allocate(word_size);

       assert(result != NULL, "we just allocate out of an empty region "

              "so allocation should have been successful");

       assert(is_in(result), "result should be in the heap");

       _cur_alloc_region = new_cur_alloc_region;

       if (!at_safepoint) {

         Heap_lock->unlock();

       }

       // do the dirtying, if necessary, after we release the Heap_lock

       if (do_dirtying) {

         dirty_young_block(result, word_size);

       }

       return result;

     }

   }

   assert(_cur_alloc_region == NULL, "we failed to allocate a new current "

          "alloc region, it should still be NULL");

   assert_heap_locked_or_at_safepoint();

   return NULL;

 }

 // See the comment in the .hpp file about the locking protocol and

 // assumptions of this method (and other related ones).

 HeapWord*

 G1CollectedHeap::attempt_allocation_slow(size_t word_size) {

   assert_heap_locked_and_not_at_safepoint();

   assert(!isHumongous(word_size), "attempt_allocation_slow() should not be "

          "used for humongous allocations");

   // We will loop while succeeded is false, which means that we tried

   // to do a collection, but the VM op did not succeed. So, when we

   // exit the loop, either one of the allocation attempts was

   // successful, or we succeeded in doing the VM op but which was

   // unable to allocate after the collection.

   for (int try_count = 1; /* we'll return or break */; try_count += 1) {

     bool succeeded = true;

     {

       // We may have concurrent cleanup working at the time. Wait for

       // it to complete. In the future we would probably want to make

       // the concurrent cleanup truly concurrent by decoupling it from

       // the allocation. This will happen in the near future as part

       // of CR 6977804 which will revamp the operation of the free

       // region list. The fact that wait_for_cleanup_complete() will

       // do a wait() means that we'll give up the Heap_lock. So, it's

       // possible that when we exit wait_for_cleanup_complete() we

       // might be able to allocate successfully (since somebody else

       // might have done a collection meanwhile). So, we'll attempt to

       // allocate again, just in case. When we make cleanup truly

       // concurrent with allocation, we should remove this allocation

       // attempt as it's redundant (we only reach here after an

       // allocation attempt has been unsuccessful).

       wait_for_cleanup_complete();

       HeapWord* result = attempt_allocation(word_size);

       if (result != NULL) {

         assert_heap_not_locked();

         return result;

       }

     }

     if (GC_locker::is_active_and_needs_gc()) {

       // We are locked out of GC because of the GC locker. We can

       // allocate a new region only if we can expand the young gen.

       if (g1_policy()->can_expand_young_list()) {

         // Yes, we are allowed to expand the young gen. Let's try to

         // allocate a new current alloc region.

         HeapWord* result =

           replace_cur_alloc_region_and_allocate(word_size,

                                                 false, /* at_safepoint */

                                                 true,  /* do_dirtying */

                                                 true   /* can_expand */);

         if (result != NULL) {

           assert_heap_not_locked();

           return result;

         }

       }

       // We could not expand the young gen further (or we could but we

       // failed to allocate a new region). We'll stall until the GC

       // locker forces a GC.

       // If this thread is not in a jni critical section, we stall

       // the requestor until the critical section has cleared and

       // GC allowed. When the critical section clears, a GC is

       // initiated by the last thread exiting the critical section; so

       // we retry the allocation sequence from the beginning of the loop,

       // rather than causing more, now probably unnecessary, GC attempts.

       JavaThread* jthr = JavaThread::current();

       assert(jthr != NULL, "sanity");

       if (!jthr->in_critical()) {

         MutexUnlocker mul(Heap_lock);

         GC_locker::stall_until_clear();

         // We'll then fall off the end of the ("if GC locker active")

         // if-statement and retry the allocation further down in the

         // loop.

       } else {

         if (CheckJNICalls) {

           fatal("Possible deadlock due to allocating while"

                 " in jni critical section");

         }

         return NULL;

       }

     } else {

       // We are not locked out. So, let's try to do a GC. The VM op

       // will retry the allocation before it completes.

       // Read the GC count while holding the Heap_lock

       unsigned int gc_count_before = SharedHeap::heap()->total_collections();

       Heap_lock->unlock();

       HeapWord* result =

         do_collection_pause(word_size, gc_count_before, &succeeded);

       assert_heap_not_locked();

       if (result != NULL) {

         assert(succeeded, "the VM op should have succeeded");

         // Allocations that take place on VM operations do not do any

         // card dirtying and we have to do it here.

         dirty_young_block(result, word_size);

         return result;

       }

       Heap_lock->lock();

     }

     assert_heap_locked();

     // We can reach here when we were unsuccessful in doing a GC,

     // because another thread beat us to it, or because we were locked

     // out of GC due to the GC locker. In either case a new alloc

     // region might be available so we will retry the allocation.

     HeapWord* result = attempt_allocation(word_size);

     if (result != NULL) {

       assert_heap_not_locked();

       return result;

     }

     // So far our attempts to allocate failed. The only time we'll go

     // around the loop and try again is if we tried to do a GC and the

     // VM op that we tried to schedule was not successful because

     // another thread beat us to it. If that happened it's possible

     // that by the time we grabbed the Heap_lock again and tried to

     // allocate other threads filled up the young generation, which

     // means that the allocation attempt after the GC also failed. So,

     // it's worth trying to schedule another GC pause.

     if (succeeded) {

       break;

     }

     // Give a warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::attempt_allocation_slow() "

               "retries %d times", try_count);

     }

   }

   assert_heap_locked();

   return NULL;

 }

 // See the comment in the .hpp file about the locking protocol and

 // assumptions of this method (and other related ones).

 HeapWord*

 G1CollectedHeap::attempt_allocation_humongous(size_t word_size,

                                               bool at_safepoint) {

   // This is the method that will allocate a humongous object. All

   // allocation paths that attempt to allocate a humongous object

   // should eventually reach here. Currently, the only paths are from

   // mem_allocate() and attempt_allocation_at_safepoint().

   assert_heap_locked_or_at_safepoint();

   assert(isHumongous(word_size), "attempt_allocation_humongous() "

          "should only be used for humongous allocations");

   assert(SafepointSynchronize::is_at_safepoint() == at_safepoint,

          "at_safepoint and is_at_safepoint() should be a tautology");

   HeapWord* result = NULL;

   // We will loop while succeeded is false, which means that we tried

   // to do a collection, but the VM op did not succeed. So, when we

   // exit the loop, either one of the allocation attempts was

   // successful, or we succeeded in doing the VM op but which was

   // unable to allocate after the collection.

   for (int try_count = 1; /* we'll return or break */; try_count += 1) {

     bool succeeded = true;

     // Given that humongous objects are not allocated in young

     // regions, we'll first try to do the allocation without doing a

     // collection hoping that there's enough space in the heap.

     result = humongous_obj_allocate(word_size);

     assert(_cur_alloc_region == NULL || !_cur_alloc_region->isHumongous(),

            "catch a regression of this bug.");

     if (result != NULL) {

       if (!at_safepoint) {

         // If we're not at a safepoint, unlock the Heap_lock.

         Heap_lock->unlock();

       }

       return result;

     }

     // If we failed to allocate the humongous object, we should try to

     // do a collection pause (if we're allowed) in case it reclaims

     // enough space for the allocation to succeed after the pause.

     if (!at_safepoint) {

       // Read the GC count while holding the Heap_lock

       unsigned int gc_count_before = SharedHeap::heap()->total_collections();

       // If we're allowed to do a collection we're not at a

       // safepoint, so it is safe to unlock the Heap_lock.

       Heap_lock->unlock();

       result = do_collection_pause(word_size, gc_count_before, &succeeded);

       assert_heap_not_locked();

       if (result != NULL) {

         assert(succeeded, "the VM op should have succeeded");

         return result;

       }

       // If we get here, the VM operation either did not succeed

       // (i.e., another thread beat us to it) or it succeeded but

       // failed to allocate the object.

       // If we're allowed to do a collection we're not at a

       // safepoint, so it is safe to lock the Heap_lock.

       Heap_lock->lock();

     }

     assert(result == NULL, "otherwise we should have exited the loop earlier");

     // So far our attempts to allocate failed. The only time we'll go

     // around the loop and try again is if we tried to do a GC and the

     // VM op that we tried to schedule was not successful because

     // another thread beat us to it. That way it's possible that some

     // space was freed up by the thread that successfully scheduled a

     // GC. So it's worth trying to allocate again.

     if (succeeded) {

       break;

     }

     // Give a warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::attempt_allocation_humongous "

               "retries %d times", try_count);

     }

   }

   assert_heap_locked_or_at_safepoint();

   return NULL;

 }

 HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size,

                                            bool expect_null_cur_alloc_region) {

   assert_at_safepoint();

   assert(_cur_alloc_region == NULL || !expect_null_cur_alloc_region,

          err_msg("the current alloc region was unexpectedly found "

                  "to be non-NULL, cur alloc region: "PTR_FORMAT" "

                  "expect_null_cur_alloc_region: %d word_size: "SIZE_FORMAT,

                  _cur_alloc_region, expect_null_cur_alloc_region, word_size));

   if (!isHumongous(word_size)) {

     if (!expect_null_cur_alloc_region) {

       HeapRegion* cur_alloc_region = _cur_alloc_region;

       if (cur_alloc_region != NULL) {

         // This allocate method does BOT updates and we don't need them in

         // the young generation. This will be fixed in the near future by

         // CR 6994297.

         HeapWord* result = cur_alloc_region->allocate(word_size);

         if (result != NULL) {

           assert(is_in(result), "result should be in the heap");

           // We will not do any dirtying here. This is guaranteed to be

           // called during a safepoint and the thread that scheduled the

           // pause will do the dirtying if we return a non-NULL result.

           return result;

         }

         retire_cur_alloc_region_common(cur_alloc_region);

       }

     }

     assert(_cur_alloc_region == NULL,

            "at this point we should have no cur alloc region");

     return replace_cur_alloc_region_and_allocate(word_size,

                                                  true, /* at_safepoint */

                                                  false /* do_dirtying */,

                                                  false /* can_expand */);

   } else {

     return attempt_allocation_humongous(word_size,

                                         true /* at_safepoint */);

   }

   ShouldNotReachHere();

 }

 HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) {

   assert_heap_not_locked_and_not_at_safepoint();

   assert(!isHumongous(word_size), "we do not allow TLABs of humongous size");

   Heap_lock->lock();

   // First attempt: try allocating out of the current alloc region or

   // after replacing the current alloc region.

   HeapWord* result = attempt_allocation(word_size);

   if (result != NULL) {

     assert_heap_not_locked();

     return result;

   }

   assert_heap_locked();

   // Second attempt: go into the even slower path where we might

   // try to schedule a collection.

   result = attempt_allocation_slow(word_size);

   if (result != NULL) {

     assert_heap_not_locked();

     return result;

   }

   assert_heap_locked();

   Heap_lock->unlock();

   return NULL;

 }

 HeapWord*

 G1CollectedHeap::mem_allocate(size_t word_size,

                               bool   is_noref,

                               bool   is_tlab,

                               bool*  gc_overhead_limit_was_exceeded) {

   assert_heap_not_locked_and_not_at_safepoint();

   assert(!is_tlab, "mem_allocate() this should not be called directly "

          "to allocate TLABs");

   // Loop until the allocation is satisified,

   // or unsatisfied after GC.

   for (int try_count = 1; /* we'll return */; try_count += 1) {

     unsigned int gc_count_before;

     {

       Heap_lock->lock();

       if (!isHumongous(word_size)) {

         // First attempt: try allocating out of the current alloc

         // region or after replacing the current alloc region.

         HeapWord* result = attempt_allocation(word_size);

         if (result != NULL) {

           assert_heap_not_locked();

           return result;

         }

         assert_heap_locked();

         // Second attempt: go into the even slower path where we might

         // try to schedule a collection.

         result = attempt_allocation_slow(word_size);

         if (result != NULL) {

           assert_heap_not_locked();

           return result;

         }

       } else {

         HeapWord* result = attempt_allocation_humongous(word_size,

                                                      false /* at_safepoint */);

         if (result != NULL) {

           assert_heap_not_locked();

           return result;

         }

       }

       assert_heap_locked();

       // Read the gc count while the heap lock is held.

       gc_count_before = SharedHeap::heap()->total_collections();

       // We cannot be at a safepoint, so it is safe to unlock the Heap_lock

       Heap_lock->unlock();

     }

     // Create the garbage collection operation...

     VM_G1CollectForAllocation op(gc_count_before, word_size);

     // ...and get the VM thread to execute it.

     VMThread::execute(&op);

     assert_heap_not_locked();

     if (op.prologue_succeeded() && op.pause_succeeded()) {

       // If the operation was successful we'll return the result even

       // if it is NULL. If the allocation attempt failed immediately

       // after a Full GC, it's unlikely we'll be able to allocate now.

       HeapWord* result = op.result();

       if (result != NULL && !isHumongous(word_size)) {

         // Allocations that take place on VM operations do not do any

         // card dirtying and we have to do it here. We only have to do

         // this for non-humongous allocations, though.

         dirty_young_block(result, word_size);

       }

       return result;

     } else {

       assert(op.result() == NULL,

              "the result should be NULL if the VM op did not succeed");

     }

     // Give a warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::mem_allocate retries %d times", try_count);

     }

   }

   ShouldNotReachHere();

 }

 void G1CollectedHeap::abandon_cur_alloc_region() {

   if (_cur_alloc_region != NULL) {

     // We're finished with the _cur_alloc_region.

     if (_cur_alloc_region->is_empty()) {

       _free_regions++;

       free_region(_cur_alloc_region);

     } else {

       // As we're builing (at least the young portion) of the collection

       // set incrementally we'll add the current allocation region to

       // the collection set here.

       if (_cur_alloc_region->is_young()) {

         g1_policy()->add_region_to_incremental_cset_lhs(_cur_alloc_region);

       }

       _summary_bytes_used += _cur_alloc_region->used();

     }

     _cur_alloc_region = NULL;

   }

 }

 void G1CollectedHeap::abandon_gc_alloc_regions() {

   // first, make sure that the GC alloc region list is empty (it should!)

   assert(_gc_alloc_region_list == NULL, "invariant");

   release_gc_alloc_regions(true /* totally */);

 }

 class PostMCRemSetClearClosure: public HeapRegionClosure {

   ModRefBarrierSet* _mr_bs;

 public:

   PostMCRemSetClearClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}

   bool doHeapRegion(HeapRegion* r) {

     r->reset_gc_time_stamp();

     if (r->continuesHumongous())

       return false;

     HeapRegionRemSet* hrrs = r->rem_set();

     if (hrrs != NULL) hrrs->clear();

     // You might think here that we could clear just the cards

     // corresponding to the used region.  But no: if we leave a dirty card

     // in a region we might allocate into, then it would prevent that card

     // from being enqueued, and cause it to be missed.

     // Re: the performance cost: we shouldn't be doing full GC anyway!

     _mr_bs->clear(MemRegion(r->bottom(), r->end()));

     return false;

   }

 };

 class PostMCRemSetInvalidateClosure: public HeapRegionClosure {

   ModRefBarrierSet* _mr_bs;

 public:

   PostMCRemSetInvalidateClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}

   bool doHeapRegion(HeapRegion* r) {

     if (r->continuesHumongous()) return false;

     if (r->used_region().word_size() != 0) {

       _mr_bs->invalidate(r->used_region(), true /*whole heap*/);

     }

     return false;

   }

 };

 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {

   G1CollectedHeap*   _g1h;

   UpdateRSOopClosure _cl;

   int                _worker_i;

 public:

   RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, int worker_i = 0) :

     _cl(g1->g1_rem_set(), worker_i),

     _worker_i(worker_i),

     _g1h(g1)

   { }

   bool doHeapRegion(HeapRegion* r) {

     if (!r->continuesHumongous()) {

       _cl.set_from(r);

       r->oop_iterate(&_cl);

     }

     return false;

   }

 };

 class ParRebuildRSTask: public AbstractGangTask {

   G1CollectedHeap* _g1;

 public:

   ParRebuildRSTask(G1CollectedHeap* g1)

     : AbstractGangTask("ParRebuildRSTask"),

       _g1(g1)

   { }

   void work(int i) {

     RebuildRSOutOfRegionClosure rebuild_rs(_g1, i);

     _g1->heap_region_par_iterate_chunked(&rebuild_rs, i,

                                          HeapRegion::RebuildRSClaimValue);

   }

 };

 bool G1CollectedHeap::do_collection(bool explicit_gc,

                                     bool clear_all_soft_refs,

                                     size_t word_size) {

   if (GC_locker::check_active_before_gc()) {

     return false;

   }

   SvcGCMarker sgcm(SvcGCMarker::FULL);

   ResourceMark rm;

   if (PrintHeapAtGC) {

     Universe::print_heap_before_gc();

   }

   assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");

   assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread");

   const bool do_clear_all_soft_refs = clear_all_soft_refs ||

                            collector_policy()->should_clear_all_soft_refs();

   ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy());

   {

     IsGCActiveMark x;

     // Timing

     bool system_gc = (gc_cause() == GCCause::_java_lang_system_gc);

     assert(!system_gc || explicit_gc, "invariant");

     gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);

     TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);

     TraceTime t(system_gc ? "Full GC (System.gc())" : "Full GC",

                 PrintGC, true, gclog_or_tty);

     TraceMemoryManagerStats tms(true /* fullGC */);

     double start = os::elapsedTime();

     g1_policy()->record_full_collection_start();

     gc_prologue(true);

     increment_total_collections(true /* full gc */);

     size_t g1h_prev_used = used();

     assert(used() == recalculate_used(), "Should be equal");

     if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       prepare_for_verify();

       gclog_or_tty->print(" VerifyBeforeGC:");

       Universe::verify(true);

     }

     assert(regions_accounted_for(), "Region leakage!");

     COMPILER2_PRESENT(DerivedPointerTable::clear());

     // We want to discover references, but not process them yet.

     // This mode is disabled in

     // instanceRefKlass::process_discovered_references if the

     // generation does some collection work, or

     // instanceRefKlass::enqueue_discovered_references if the

     // generation returns without doing any work.

     ref_processor()->disable_discovery();

     ref_processor()->abandon_partial_discovery();

     ref_processor()->verify_no_references_recorded();

     // Abandon current iterations of concurrent marking and concurrent

     // refinement, if any are in progress.

     concurrent_mark()->abort();

     // Make sure we'll choose a new allocation region afterwards.

     abandon_cur_alloc_region();

     abandon_gc_alloc_regions();

     assert(_cur_alloc_region == NULL, "Invariant.");

     g1_rem_set()->cleanupHRRS();

     tear_down_region_lists();

     set_used_regions_to_need_zero_fill();

     // We may have added regions to the current incremental collection

     // set between the last GC or pause and now. We need to clear the

     // incremental collection set and then start rebuilding it afresh

     // after this full GC.

     abandon_collection_set(g1_policy()->inc_cset_head());

     g1_policy()->clear_incremental_cset();

     g1_policy()->stop_incremental_cset_building();

     if (g1_policy()->in_young_gc_mode()) {

       empty_young_list();

       g1_policy()->set_full_young_gcs(true);

     }

     // See the comment in G1CollectedHeap::ref_processing_init() about

     // how reference processing currently works in G1.

     // Temporarily make reference _discovery_ single threaded (non-MT).

     ReferenceProcessorMTMutator rp_disc_ser(ref_processor(), false);

     // Temporarily make refs discovery atomic

     ReferenceProcessorAtomicMutator rp_disc_atomic(ref_processor(), true);

     // Temporarily clear _is_alive_non_header

     ReferenceProcessorIsAliveMutator rp_is_alive_null(ref_processor(), NULL);

     ref_processor()->enable_discovery();

     ref_processor()->setup_policy(do_clear_all_soft_refs);

     // Do collection work

     {

       HandleMark hm;  // Discard invalid handles created during gc

       G1MarkSweep::invoke_at_safepoint(ref_processor(), do_clear_all_soft_refs);

     }

     // Because freeing humongous regions may have added some unclean

     // regions, it is necessary to tear down again before rebuilding.

     tear_down_region_lists();

     rebuild_region_lists();

     _summary_bytes_used = recalculate_used();

     ref_processor()->enqueue_discovered_references();

     COMPILER2_PRESENT(DerivedPointerTable::update_pointers());

     MemoryService::track_memory_usage();

     if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       gclog_or_tty->print(" VerifyAfterGC:");

       prepare_for_verify();

       Universe::verify(false);

     }

     NOT_PRODUCT(ref_processor()->verify_no_references_recorded());

     reset_gc_time_stamp();

     // Since everything potentially moved, we will clear all remembered

     // sets, and clear all cards.  Later we will rebuild remebered

     // sets. We will also reset the GC time stamps of the regions.

     PostMCRemSetClearClosure rs_clear(mr_bs());

     heap_region_iterate(&rs_clear);

     // Resize the heap if necessary.

     resize_if_necessary_after_full_collection(explicit_gc ? 0 : word_size);

     if (_cg1r->use_cache()) {

       _cg1r->clear_and_record_card_counts();

       _cg1r->clear_hot_cache();

     }

     // Rebuild remembered sets of all regions.

     if (G1CollectedHeap::use_parallel_gc_threads()) {

       ParRebuildRSTask rebuild_rs_task(this);

       assert(check_heap_region_claim_values(

              HeapRegion::InitialClaimValue), "sanity check");

       set_par_threads(workers()->total_workers());

       workers()->run_task(&rebuild_rs_task);

       set_par_threads(0);

       assert(check_heap_region_claim_values(

              HeapRegion::RebuildRSClaimValue), "sanity check");

       reset_heap_region_claim_values();

     } else {

       RebuildRSOutOfRegionClosure rebuild_rs(this);

       heap_region_iterate(&rebuild_rs);

     }

     if (PrintGC) {

       print_size_transition(gclog_or_tty, g1h_prev_used, used(), capacity());

     }

     if (true) { // FIXME

       // Ask the permanent generation to adjust size for full collections

       perm()->compute_new_size();

     }

     // Start a new incremental collection set for the next pause

     assert(g1_policy()->collection_set() == NULL, "must be");

     g1_policy()->start_incremental_cset_building();

     // Clear the _cset_fast_test bitmap in anticipation of adding

     // regions to the incremental collection set for the next

     // evacuation pause.

     clear_cset_fast_test();

     double end = os::elapsedTime();

     g1_policy()->record_full_collection_end();

 #ifdef TRACESPINNING

     ParallelTaskTerminator::print_termination_counts();

 #endif

     gc_epilogue(true);

     // Discard all rset updates

     JavaThread::dirty_card_queue_set().abandon_logs();

     assert(!G1DeferredRSUpdate

            || (G1DeferredRSUpdate && (dirty_card_queue_set().completed_buffers_num() == 0)), "Should not be any");

     assert(regions_accounted_for(), "Region leakage!");

   }

   if (g1_policy()->in_young_gc_mode()) {

     _young_list->reset_sampled_info();

     // At this point there should be no regions in the

     // entire heap tagged as young.

     assert( check_young_list_empty(true /* check_heap */),

             "young list should be empty at this point");

   }

   // Update the number of full collections that have been completed.

   increment_full_collections_completed(false /* concurrent */);

   if (PrintHeapAtGC) {

     Universe::print_heap_after_gc();

   }

   return true;

 }

 void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {

   // do_collection() will return whether it succeeded in performing

   // the GC. Currently, there is no facility on the

   // do_full_collection() API to notify the caller than the collection

   // did not succeed (e.g., because it was locked out by the GC

   // locker). So, right now, we'll ignore the return value.

   bool dummy = do_collection(true,                /* explicit_gc */

                              clear_all_soft_refs,

                              0                    /* word_size */);

 }

 // This code is mostly copied from TenuredGeneration.

 void

 G1CollectedHeap::

 resize_if_necessary_after_full_collection(size_t word_size) {

   assert(MinHeapFreeRatio <= MaxHeapFreeRatio, "sanity check");

   // Include the current allocation, if any, and bytes that will be

   // pre-allocated to support collections, as "used".

   const size_t used_after_gc = used();

   const size_t capacity_after_gc = capacity();

   const size_t free_after_gc = capacity_after_gc - used_after_gc;

   // This is enforced in arguments.cpp.

   assert(MinHeapFreeRatio <= MaxHeapFreeRatio,

          "otherwise the code below doesn't make sense");

   // We don't have floating point command-line arguments

   const double minimum_free_percentage = (double) MinHeapFreeRatio / 100.0;

   const double maximum_used_percentage = 1.0 - minimum_free_percentage;

   const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100.0;

   const double minimum_used_percentage = 1.0 - maximum_free_percentage;

   const size_t min_heap_size = collector_policy()->min_heap_byte_size();

   const size_t max_heap_size = collector_policy()->max_heap_byte_size();

   // We have to be careful here as these two calculations can overflow

   // 32-bit size_t's.

   double used_after_gc_d = (double) used_after_gc;

   double minimum_desired_capacity_d = used_after_gc_d / maximum_used_percentage;

   double maximum_desired_capacity_d = used_after_gc_d / minimum_used_percentage;

   // Let's make sure that they are both under the max heap size, which

   // by default will make them fit into a size_t.

   double desired_capacity_upper_bound = (double) max_heap_size;

   minimum_desired_capacity_d = MIN2(minimum_desired_capacity_d,

                                     desired_capacity_upper_bound);

   maximum_desired_capacity_d = MIN2(maximum_desired_capacity_d,

                                     desired_capacity_upper_bound);

   // We can now safely turn them into size_t's.

   size_t minimum_desired_capacity = (size_t) minimum_desired_capacity_d;

   size_t maximum_desired_capacity = (size_t) maximum_desired_capacity_d;

   // This assert only makes sense here, before we adjust them

   // with respect to the min and max heap size.

   assert(minimum_desired_capacity <= maximum_desired_capacity,

          err_msg("minimum_desired_capacity = "SIZE_FORMAT", "

                  "maximum_desired_capacity = "SIZE_FORMAT,

                  minimum_desired_capacity, maximum_desired_capacity));

   // Should not be greater than the heap max size. No need to adjust

   // it with respect to the heap min size as it's a lower bound (i.e.,

   // we'll try to make the capacity larger than it, not smaller).

   minimum_desired_capacity = MIN2(minimum_desired_capacity, max_heap_size);

   // Should not be less than the heap min size. No need to adjust it

   // with respect to the heap max size as it's an upper bound (i.e.,

   // we'll try to make the capacity smaller than it, not greater).

   maximum_desired_capacity =  MAX2(maximum_desired_capacity, min_heap_size);

   if (PrintGC && Verbose) {

     const double free_percentage =

       (double) free_after_gc / (double) capacity_after_gc;

     gclog_or_tty->print_cr("Computing new size after full GC ");

     gclog_or_tty->print_cr("  "

                            "  minimum_free_percentage: %6.2f",

                            minimum_free_percentage);

     gclog_or_tty->print_cr("  "

                            "  maximum_free_percentage: %6.2f",

                            maximum_free_percentage);

     gclog_or_tty->print_cr("  "

                            "  capacity: %6.1fK"

                            "  minimum_desired_capacity: %6.1fK"

                            "  maximum_desired_capacity: %6.1fK",

                            (double) capacity_after_gc / (double) K,

                            (double) minimum_desired_capacity / (double) K,

                            (double) maximum_desired_capacity / (double) K);

     gclog_or_tty->print_cr("  "

                            "  free_after_gc: %6.1fK"

                            "  used_after_gc: %6.1fK",

                            (double) free_after_gc / (double) K,

                            (double) used_after_gc / (double) K);

     gclog_or_tty->print_cr("  "

                            "   free_percentage: %6.2f",

                            free_percentage);

   }

   if (capacity_after_gc < minimum_desired_capacity) {

     // Don't expand unless it's significant

     size_t expand_bytes = minimum_desired_capacity - capacity_after_gc;

     expand(expand_bytes);

     if (PrintGC && Verbose) {

       gclog_or_tty->print_cr("  "

                              "  expanding:"

                              "  max_heap_size: %6.1fK"

                              "  minimum_desired_capacity: %6.1fK"

                              "  expand_bytes: %6.1fK",

                              (double) max_heap_size / (double) K,

                              (double) minimum_desired_capacity / (double) K,

                              (double) expand_bytes / (double) K);

     }

     // No expansion, now see if we want to shrink

   } else if (capacity_after_gc > maximum_desired_capacity) {

     // Capacity too large, compute shrinking size

     size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity;

     shrink(shrink_bytes);

     if (PrintGC && Verbose) {

       gclog_or_tty->print_cr("  "

                              "  shrinking:"

                              "  min_heap_size: %6.1fK"

                              "  maximum_desired_capacity: %6.1fK"

                              "  shrink_bytes: %6.1fK",

                              (double) min_heap_size / (double) K,

                              (double) maximum_desired_capacity / (double) K,

                              (double) shrink_bytes / (double) K);

     }

   }

 }

 HeapWord*

 G1CollectedHeap::satisfy_failed_allocation(size_t word_size,

                                            bool* succeeded) {

   assert(SafepointSynchronize::is_at_safepoint(),

          "satisfy_failed_allocation() should only be called at a safepoint");

   assert(Thread::current()->is_VM_thread(),

          "satisfy_failed_allocation() should only be called by the VM thread");

   *succeeded = true;

   // Let's attempt the allocation first.

   HeapWord* result = attempt_allocation_at_safepoint(word_size,

                                      false /* expect_null_cur_alloc_region */);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   // In a G1 heap, we're supposed to keep allocation from failing by

   // incremental pauses.  Therefore, at least for now, we'll favor

   // expansion over collection.  (This might change in the future if we can

   // do something smarter than full collection to satisfy a failed alloc.)

   result = expand_and_allocate(word_size);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   // Expansion didn't work, we'll try to do a Full GC.

   bool gc_succeeded = do_collection(false, /* explicit_gc */

                                     false, /* clear_all_soft_refs */

                                     word_size);

   if (!gc_succeeded) {

     *succeeded = false;

     return NULL;

   }

   // Retry the allocation

   result = attempt_allocation_at_safepoint(word_size,

                                       true /* expect_null_cur_alloc_region */);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   // Then, try a Full GC that will collect all soft references.

   gc_succeeded = do_collection(false, /* explicit_gc */

                                true,  /* clear_all_soft_refs */

                                word_size);

   if (!gc_succeeded) {

     *succeeded = false;

     return NULL;

   }

   // Retry the allocation once more

   result = attempt_allocation_at_safepoint(word_size,

                                       true /* expect_null_cur_alloc_region */);

   if (result != NULL) {

     assert(*succeeded, "sanity");

     return result;

   }

   assert(!collector_policy()->should_clear_all_soft_refs(),

          "Flag should have been handled and cleared prior to this point");

   // What else?  We might try synchronous finalization later.  If the total

   // space available is large enough for the allocation, then a more

   // complete compaction phase than we've tried so far might be

   // appropriate.

   assert(*succeeded, "sanity");

   return NULL;

 }

 // Attempting to expand the heap sufficiently

 // to support an allocation of the given "word_size".  If

 // successful, perform the allocation and return the address of the

 // allocated block, or else "NULL".

 HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size) {

   assert(SafepointSynchronize::is_at_safepoint(),

          "expand_and_allocate() should only be called at a safepoint");

   assert(Thread::current()->is_VM_thread(),

          "expand_and_allocate() should only be called by the VM thread");

   size_t expand_bytes = word_size * HeapWordSize;

   if (expand_bytes < MinHeapDeltaBytes) {

     expand_bytes = MinHeapDeltaBytes;

   }

   expand(expand_bytes);

   assert(regions_accounted_for(), "Region leakage!");

   return attempt_allocation_at_safepoint(word_size,

                                      false /* expect_null_cur_alloc_region */);

 }

 size_t G1CollectedHeap::free_region_if_totally_empty(HeapRegion* hr) {

   size_t pre_used = 0;

   size_t cleared_h_regions = 0;

   size_t freed_regions = 0;

   UncleanRegionList local_list;

   free_region_if_totally_empty_work(hr, pre_used, cleared_h_regions,

                                     freed_regions, &local_list);

   finish_free_region_work(pre_used, cleared_h_regions, freed_regions,

                           &local_list);

   return pre_used;

 }

 void

 G1CollectedHeap::free_region_if_totally_empty_work(HeapRegion* hr,

                                                    size_t& pre_used,

                                                    size_t& cleared_h,

                                                    size_t& freed_regions,

                                                    UncleanRegionList* list,

                                                    bool par) {

   assert(!hr->continuesHumongous(), "should have filtered these out");

   size_t res = 0;

   if (hr->used() > 0 && hr->garbage_bytes() == hr->used() &&

       !hr->is_young()) {

     if (G1PolicyVerbose > 0)

       gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)"

                                                                                " during cleanup", hr, hr->used());

     free_region_work(hr, pre_used, cleared_h, freed_regions, list, par);

   }

 }

 // FIXME: both this and shrink could probably be more efficient by

 // doing one "VirtualSpace::expand_by" call rather than several.

 void G1CollectedHeap::expand(size_t expand_bytes) {

   size_t old_mem_size = _g1_storage.committed_size();

   // We expand by a minimum of 1K.

   expand_bytes = MAX2(expand_bytes, (size_t)K);

   size_t aligned_expand_bytes =

     ReservedSpace::page_align_size_up(expand_bytes);

   aligned_expand_bytes = align_size_up(aligned_expand_bytes,

                                        HeapRegion::GrainBytes);

   expand_bytes = aligned_expand_bytes;

   while (expand_bytes > 0) {

     HeapWord* base = (HeapWord*)_g1_storage.high();

     // Commit more storage.

     bool successful = _g1_storage.expand_by(HeapRegion::GrainBytes);

     if (!successful) {

         expand_bytes = 0;

     } else {

       expand_bytes -= HeapRegion::GrainBytes;

       // Expand the committed region.

       HeapWord* high = (HeapWord*) _g1_storage.high();

       _g1_committed.set_end(high);

       // Create a new HeapRegion.

       MemRegion mr(base, high);

       bool is_zeroed = !_g1_max_committed.contains(base);

       HeapRegion* hr = new HeapRegion(_bot_shared, mr, is_zeroed);

       // Now update max_committed if necessary.

       _g1_max_committed.set_end(MAX2(_g1_max_committed.end(), high));

       // Add it to the HeapRegionSeq.

       _hrs->insert(hr);

       // Set the zero-fill state, according to whether it's already

       // zeroed.

       {

         MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);

         if (is_zeroed) {

           hr->set_zero_fill_complete();

           put_free_region_on_list_locked(hr);

         } else {

           hr->set_zero_fill_needed();

           put_region_on_unclean_list_locked(hr);

         }

       }

       _free_regions++;

       // And we used up an expansion region to create it.

       _expansion_regions--;

       // Tell the cardtable about it.

       Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);

       // And the offset table as well.

       _bot_shared->resize(_g1_committed.word_size());

     }

   }

   if (Verbose && PrintGC) {

     size_t new_mem_size = _g1_storage.committed_size();

     gclog_or_tty->print_cr("Expanding garbage-first heap from %ldK by %ldK to %ldK",

                            old_mem_size/K, aligned_expand_bytes/K,

                            new_mem_size/K);

   }

 }

 void G1CollectedHeap::shrink_helper(size_t shrink_bytes)

 {

   size_t old_mem_size = _g1_storage.committed_size();

   size_t aligned_shrink_bytes =

     ReservedSpace::page_align_size_down(shrink_bytes);

   aligned_shrink_bytes = align_size_down(aligned_shrink_bytes,

                                          HeapRegion::GrainBytes);

   size_t num_regions_deleted = 0;

   MemRegion mr = _hrs->shrink_by(aligned_shrink_bytes, num_regions_deleted);

   assert(mr.end() == (HeapWord*)_g1_storage.high(), "Bad shrink!");

   if (mr.byte_size() > 0)

     _g1_storage.shrink_by(mr.byte_size());

   assert(mr.start() == (HeapWord*)_g1_storage.high(), "Bad shrink!");

   _g1_committed.set_end(mr.start());

   _free_regions -= num_regions_deleted;

   _expansion_regions += num_regions_deleted;

   // Tell the cardtable about it.

   Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);

   // And the offset table as well.

   _bot_shared->resize(_g1_committed.word_size());

   HeapRegionRemSet::shrink_heap(n_regions());

   if (Verbose && PrintGC) {

     size_t new_mem_size = _g1_storage.committed_size();

     gclog_or_tty->print_cr("Shrinking garbage-first heap from %ldK by %ldK to %ldK",

                            old_mem_size/K, aligned_shrink_bytes/K,

                            new_mem_size/K);

   }

 }

 void G1CollectedHeap::shrink(size_t shrink_bytes) {

   release_gc_alloc_regions(true /* totally */);

   tear_down_region_lists();  // We will rebuild them in a moment.

   shrink_helper(shrink_bytes);

   rebuild_region_lists();

 }

 // Public methods.

 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away

 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list

 #endif // _MSC_VER

 G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) :

   SharedHeap(policy_),

   _g1_policy(policy_),

   _dirty_card_queue_set(false),

   _into_cset_dirty_card_queue_set(false),

   _is_alive_closure(this),

   _ref_processor(NULL),

   _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)),

   _bot_shared(NULL),

   _par_alloc_during_gc_lock(Mutex::leaf, "par alloc during GC lock"),

   _objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL),

   _evac_failure_scan_stack(NULL) ,

   _mark_in_progress(false),

   _cg1r(NULL), _czft(NULL), _summary_bytes_used(0),

   _cur_alloc_region(NULL),

   _refine_cte_cl(NULL),

   _free_region_list(NULL), _free_region_list_size(0),

   _free_regions(0),

   _full_collection(false),

   _unclean_region_list(),

   _unclean_regions_coming(false),

   _young_list(new YoungList(this)),

   _gc_time_stamp(0),

   _surviving_young_words(NULL),

   _full_collections_completed(0),

   _in_cset_fast_test(NULL),

   _in_cset_fast_test_base(NULL),

   _dirty_cards_region_list(NULL) {

   _g1h = this; // To catch bugs.

   if (_process_strong_tasks == NULL || !_process_strong_tasks->valid()) {

     vm_exit_during_initialization("Failed necessary allocation.");

   }

   _humongous_object_threshold_in_words = HeapRegion::GrainWords / 2;

   int n_queues = MAX2((int)ParallelGCThreads, 1);

   _task_queues = new RefToScanQueueSet(n_queues);

   int n_rem_sets = HeapRegionRemSet::num_par_rem_sets();

   assert(n_rem_sets > 0, "Invariant.");

   HeapRegionRemSetIterator** iter_arr =

     NEW_C_HEAP_ARRAY(HeapRegionRemSetIterator*, n_queues);

   for (int i = 0; i < n_queues; i++) {

     iter_arr[i] = new HeapRegionRemSetIterator();

   }

   _rem_set_iterator = iter_arr;

   for (int i = 0; i < n_queues; i++) {

     RefToScanQueue* q = new RefToScanQueue();

     q->initialize();

     _task_queues->register_queue(i, q);

   }

   for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {

     _gc_alloc_regions[ap]          = NULL;

     _gc_alloc_region_counts[ap]    = 0;

     _retained_gc_alloc_regions[ap] = NULL;

     // by default, we do not retain a GC alloc region for each ap;

     // we'll override this, when appropriate, below

     _retain_gc_alloc_region[ap]    = false;

   }

   // We will try to remember the last half-full tenured region we

   // allocated to at the end of a collection so that we can re-use it

   // during the next collection.

   _retain_gc_alloc_region[GCAllocForTenured]  = true;

   guarantee(_task_queues != NULL, "task_queues allocation failure.");

 }

 jint G1CollectedHeap::initialize() {

   CollectedHeap::pre_initialize();

   os::enable_vtime();

   // Necessary to satisfy locking discipline assertions.

   MutexLocker x(Heap_lock);

   // While there are no constraints in the GC code that HeapWordSize

   // be any particular value, there are multiple other areas in the

   // system which believe this to be true (e.g. oop->object_size in some

   // cases incorrectly returns the size in wordSize units rather than

   // HeapWordSize).

   guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize");

   size_t init_byte_size = collector_policy()->initial_heap_byte_size();

   size_t max_byte_size = collector_policy()->max_heap_byte_size();

   // Ensure that the sizes are properly aligned.

   Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap");

   Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap");

   _cg1r = new ConcurrentG1Refine();

   // Reserve the maximum.

   PermanentGenerationSpec* pgs = collector_policy()->permanent_generation();

   // Includes the perm-gen.

   const size_t total_reserved = max_byte_size + pgs->max_size();

   char* addr = Universe::preferred_heap_base(total_reserved, Universe::UnscaledNarrowOop);

   ReservedSpace heap_rs(max_byte_size + pgs->max_size(),

                         HeapRegion::GrainBytes,

                         false /*ism*/, addr);

   if (UseCompressedOops) {

     if (addr != NULL && !heap_rs.is_reserved()) {

       // Failed to reserve at specified address - the requested memory

       // region is taken already, for example, by 'java' launcher.

       // Try again to reserver heap higher.

       addr = Universe::preferred_heap_base(total_reserved, Universe::ZeroBasedNarrowOop);

       ReservedSpace heap_rs0(total_reserved, HeapRegion::GrainBytes,

                              false /*ism*/, addr);

       if (addr != NULL && !heap_rs0.is_reserved()) {

         // Failed to reserve at specified address again - give up.

         addr = Universe::preferred_heap_base(total_reserved, Universe::HeapBasedNarrowOop);

         assert(addr == NULL, "");

         ReservedSpace heap_rs1(total_reserved, HeapRegion::GrainBytes,

                                false /*ism*/, addr);

         heap_rs = heap_rs1;

       } else {

         heap_rs = heap_rs0;

       }

     }

   }

   if (!heap_rs.is_reserved()) {

     vm_exit_during_initialization("Could not reserve enough space for object heap");

     return JNI_ENOMEM;

   }

   // It is important to do this in a way such that concurrent readers can't

   // temporarily think somethings in the heap.  (I've actually seen this

   // happen in asserts: DLD.)

   _reserved.set_word_size(0);

   _reserved.set_start((HeapWord*)heap_rs.base());

   _reserved.set_end((HeapWord*)(heap_rs.base() + heap_rs.size()));

   _expansion_regions = max_byte_size/HeapRegion::GrainBytes;

   _num_humongous_regions = 0;

   // Create the gen rem set (and barrier set) for the entire reserved region.

   _rem_set = collector_policy()->create_rem_set(_reserved, 2);

   set_barrier_set(rem_set()->bs());

   if (barrier_set()->is_a(BarrierSet::ModRef)) {

     _mr_bs = (ModRefBarrierSet*)_barrier_set;

   } else {

     vm_exit_during_initialization("G1 requires a mod ref bs.");

     return JNI_ENOMEM;

   }

   // Also create a G1 rem set.

   if (mr_bs()->is_a(BarrierSet::CardTableModRef)) {

     _g1_rem_set = new G1RemSet(this, (CardTableModRefBS*)mr_bs());

   } else {

     vm_exit_during_initialization("G1 requires a cardtable mod ref bs.");

     return JNI_ENOMEM;

   }

   // Carve out the G1 part of the heap.

   ReservedSpace g1_rs   = heap_rs.first_part(max_byte_size);

   _g1_reserved = MemRegion((HeapWord*)g1_rs.base(),

                            g1_rs.size()/HeapWordSize);

   ReservedSpace perm_gen_rs = heap_rs.last_part(max_byte_size);

   _perm_gen = pgs->init(perm_gen_rs, pgs->init_size(), rem_set());

   _g1_storage.initialize(g1_rs, 0);

   _g1_committed = MemRegion((HeapWord*)_g1_storage.low(), (size_t) 0);

   _g1_max_committed = _g1_committed;

   _hrs = new HeapRegionSeq(_expansion_regions);

   guarantee(_hrs != NULL, "Couldn't allocate HeapRegionSeq");

   guarantee(_cur_alloc_region == NULL, "from constructor");

   // 6843694 - ensure that the maximum region index can fit

   // in the remembered set structures.

   const size_t max_region_idx = ((size_t)1 << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1;

   guarantee((max_regions() - 1) <= max_region_idx, "too many regions");

   size_t max_cards_per_region = ((size_t)1 << (sizeof(CardIdx_t)*BitsPerByte-1)) - 1;

   guarantee(HeapRegion::CardsPerRegion > 0, "make sure it's initialized");

   guarantee((size_t) HeapRegion::CardsPerRegion < max_cards_per_region,

             "too many cards per region");

   _bot_shared = new G1BlockOffsetSharedArray(_reserved,

                                              heap_word_size(init_byte_size));

   _g1h = this;

    _in_cset_fast_test_length = max_regions();

    _in_cset_fast_test_base = NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);

    // We're biasing _in_cset_fast_test to avoid subtracting the

    // beginning of the heap every time we want to index; basically

    // it's the same with what we do with the card table.

    _in_cset_fast_test = _in_cset_fast_test_base -

                 ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);

    // Clear the _cset_fast_test bitmap in anticipation of adding

    // regions to the incremental collection set for the first

    // evacuation pause.

    clear_cset_fast_test();

   // Create the ConcurrentMark data structure and thread.

   // (Must do this late, so that "max_regions" is defined.)

   _cm       = new ConcurrentMark(heap_rs, (int) max_regions());

   _cmThread = _cm->cmThread();

   // ...and the concurrent zero-fill thread, if necessary.

   if (G1ConcZeroFill) {

     _czft = new ConcurrentZFThread();

   }

   // Initialize the from_card cache structure of HeapRegionRemSet.

   HeapRegionRemSet::init_heap(max_regions());

   // Now expand into the initial heap size.

   expand(init_byte_size);

   // Perform any initialization actions delegated to the policy.

   g1_policy()->init();

   g1_policy()->note_start_of_mark_thread();

   _refine_cte_cl =

     new RefineCardTableEntryClosure(ConcurrentG1RefineThread::sts(),

                                     g1_rem_set(),

                                     concurrent_g1_refine());

   JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

   JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,

                                                SATB_Q_FL_lock,

                                                G1SATBProcessCompletedThreshold,

                                                Shared_SATB_Q_lock);

   JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,

                                                 DirtyCardQ_FL_lock,

                                                 concurrent_g1_refine()->yellow_zone(),

                                                 concurrent_g1_refine()->red_zone(),

                                                 Shared_DirtyCardQ_lock);

   if (G1DeferredRSUpdate) {

     dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,

                                       DirtyCardQ_FL_lock,

                                       -1, // never trigger processing

                                       -1, // no limit on length

                                       Shared_DirtyCardQ_lock,

                                       &JavaThread::dirty_card_queue_set());

   }

   // Initialize the card queue set used to hold cards containing

   // references into the collection set.

   _into_cset_dirty_card_queue_set.initialize(DirtyCardQ_CBL_mon,

                                              DirtyCardQ_FL_lock,

                                              -1, // never trigger processing

                                              -1, // no limit on length

                                              Shared_DirtyCardQ_lock,

                                              &JavaThread::dirty_card_queue_set());

   // In case we're keeping closure specialization stats, initialize those

   // counts and that mechanism.

   SpecializationStats::clear();

   _gc_alloc_region_list = NULL;

   // Do later initialization work for concurrent refinement.

   _cg1r->init();

   return JNI_OK;

 }

 void G1CollectedHeap::ref_processing_init() {

   // Reference processing in G1 currently works as follows:

   //

   // * There is only one reference processor instance that

   //   'spans' the entire heap. It is created by the code

   //   below.

   // * Reference discovery is not enabled during an incremental

   //   pause (see 6484982).

   // * Discoverered refs are not enqueued nor are they processed

   //   during an incremental pause (see 6484982).

   // * Reference discovery is enabled at initial marking.

   // * Reference discovery is disabled and the discovered

   //   references processed etc during remarking.

   // * Reference discovery is MT (see below).

   // * Reference discovery requires a barrier (see below).

   // * Reference processing is currently not MT (see 6608385).

   // * A full GC enables (non-MT) reference discovery and

   //   processes any discovered references.

   SharedHeap::ref_processing_init();

   MemRegion mr = reserved_region();

   _ref_processor = ReferenceProcessor::create_ref_processor(

                                          mr,    // span

                                          false, // Reference discovery is not atomic

                                          true,  // mt_discovery

                                          &_is_alive_closure, // is alive closure

                                                              // for efficiency

                                          ParallelGCThreads,

                                          ParallelRefProcEnabled,

                                          true); // Setting next fields of discovered

                                                 // lists requires a barrier.

 }

 size_t G1CollectedHeap::capacity() const {

   return _g1_committed.byte_size();

 }

 void G1CollectedHeap::iterate_dirty_card_closure(CardTableEntryClosure* cl,

                                                  DirtyCardQueue* into_cset_dcq,

                                                  bool concurrent,

                                                  int worker_i) {

   // Clean cards in the hot card cache

   concurrent_g1_refine()->clean_up_cache(worker_i, g1_rem_set(), into_cset_dcq);

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   int n_completed_buffers = 0;

   while (dcqs.apply_closure_to_completed_buffer(cl, worker_i, 0, true)) {

     n_completed_buffers++;

   }

   g1_policy()->record_update_rs_processed_buffers(worker_i,

                                                   (double) n_completed_buffers);

   dcqs.clear_n_completed_buffers();

   assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!");

 }

 // Computes the sum of the storage used by the various regions.

 size_t G1CollectedHeap::used() const {

   assert(Heap_lock->owner() != NULL,

          "Should be owned on this thread's behalf.");

   size_t result = _summary_bytes_used;

   // Read only once in case it is set to NULL concurrently

   HeapRegion* hr = _cur_alloc_region;

   if (hr != NULL)

     result += hr->used();

   return result;

 }

 size_t G1CollectedHeap::used_unlocked() const {

   size_t result = _summary_bytes_used;

   return result;

 }

 class SumUsedClosure: public HeapRegionClosure {

   size_t _used;

 public:

   SumUsedClosure() : _used(0) {}

   bool doHeapRegion(HeapRegion* r) {

     if (!r->continuesHumongous()) {

       _used += r->used();

     }

     return false;

   }

   size_t result() { return _used; }

 };

 size_t G1CollectedHeap::recalculate_used() const {

   SumUsedClosure blk;

   _hrs->iterate(&blk);

   return blk.result();

 }

 #ifndef PRODUCT

 class SumUsedRegionsClosure: public HeapRegionClosure {

   size_t _num;

 public:

   SumUsedRegionsClosure() : _num(0) {}

   bool doHeapRegion(HeapRegion* r) {

     if (r->continuesHumongous() || r->used() > 0 || r->is_gc_alloc_region()) {

       _num += 1;

     }

     return false;

   }

   size_t result() { return _num; }

 };

 size_t G1CollectedHeap::recalculate_used_regions() const {

   SumUsedRegionsClosure blk;

   _hrs->iterate(&blk);

   return blk.result();

 }

 #endif // PRODUCT

 size_t G1CollectedHeap::unsafe_max_alloc() {

   if (_free_regions > 0) return HeapRegion::GrainBytes;

   // otherwise, is there space in the current allocation region?

   // We need to store the current allocation region in a local variable

   // here. The problem is that this method doesn't take any locks and

   // there may be other threads which overwrite the current allocation

   // region field. attempt_allocation(), for example, sets it to NULL

   // and this can happen *after* the NULL check here but before the call

   // to free(), resulting in a SIGSEGV. Note that this doesn't appear

   // to be a problem in the optimized build, since the two loads of the

   // current allocation region field are optimized away.

   HeapRegion* car = _cur_alloc_region;

   // FIXME: should iterate over all regions?

   if (car == NULL) {

     return 0;

   }

   return car->free();

 }

 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {

   return

     ((cause == GCCause::_gc_locker           && GCLockerInvokesConcurrent) ||

      (cause == GCCause::_java_lang_system_gc && ExplicitGCInvokesConcurrent));

 }

 void G1CollectedHeap::increment_full_collections_completed(bool concurrent) {

   MonitorLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag);

   // We assume that if concurrent == true, then the caller is a

   // concurrent thread that was joined the Suspendible Thread

   // Set. If there's ever a cheap way to check this, we should add an

   // assert here.

   // We have already incremented _total_full_collections at the start

   // of the GC, so total_full_collections() represents how many full

   // collections have been started.

   unsigned int full_collections_started = total_full_collections();

   // Given that this method is called at the end of a Full GC or of a

   // concurrent cycle, and those can be nested (i.e., a Full GC can

   // interrupt a concurrent cycle), the number of full collections

   // completed should be either one (in the case where there was no

   // nesting) or two (when a Full GC interrupted a concurrent cycle)

   // behind the number of full collections started.

   // This is the case for the inner caller, i.e. a Full GC.

   assert(concurrent ||

          (full_collections_started == _full_collections_completed + 1) ||

          (full_collections_started == _full_collections_completed + 2),

          err_msg("for inner caller (Full GC): full_collections_started = %u "

                  "is inconsistent with _full_collections_completed = %u",

                  full_collections_started, _full_collections_completed));

   // This is the case for the outer caller, i.e. the concurrent cycle.

   assert(!concurrent ||

          (full_collections_started == _full_collections_completed + 1),

          err_msg("for outer caller (concurrent cycle): "

                  "full_collections_started = %u "

                  "is inconsistent with _full_collections_completed = %u",

                  full_collections_started, _full_collections_completed));

   _full_collections_completed += 1;

   // We need to clear the "in_progress" flag in the CM thread before

   // we wake up any waiters (especially when ExplicitInvokesConcurrent

   // is set) so that if a waiter requests another System.gc() it doesn't

   // incorrectly see that a marking cyle is still in progress.

   if (concurrent) {

     _cmThread->clear_in_progress();

   }

   // This notify_all() will ensure that a thread that called

   // System.gc() with (with ExplicitGCInvokesConcurrent set or not)

   // and it's waiting for a full GC to finish will be woken up. It is

   // waiting in VM_G1IncCollectionPause::doit_epilogue().

   FullGCCount_lock->notify_all();

 }

 void G1CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) {

   assert(Thread::current()->is_VM_thread(), "Precondition#1");

   assert(Heap_lock->is_locked(), "Precondition#2");

   GCCauseSetter gcs(this, cause);

   switch (cause) {

     case GCCause::_heap_inspection:

     case GCCause::_heap_dump: {

       HandleMark hm;

       do_full_collection(false);         // don't clear all soft refs

       break;

     }

     default: // XXX FIX ME

       ShouldNotReachHere(); // Unexpected use of this function

   }

 }

 void G1CollectedHeap::collect(GCCause::Cause cause) {

   // The caller doesn't have the Heap_lock

   assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock");

   unsigned int gc_count_before;

   unsigned int full_gc_count_before;

   {

     MutexLocker ml(Heap_lock);

     // Don't want to do a GC until cleanup is completed. This

     // limitation will be removed in the near future when the

     // operation of the free region list is revamped as part of

     // CR 6977804.

     wait_for_cleanup_complete();

     // Read the GC count while holding the Heap_lock

     gc_count_before = SharedHeap::heap()->total_collections();

     full_gc_count_before = SharedHeap::heap()->total_full_collections();

   }

   if (should_do_concurrent_full_gc(cause)) {

     // Schedule an initial-mark evacuation pause that will start a

     // concurrent cycle. We're setting word_size to 0 which means that

     // we are not requesting a post-GC allocation.

     VM_G1IncCollectionPause op(gc_count_before,

                                0,     /* word_size */

                                true,  /* should_initiate_conc_mark */

                                g1_policy()->max_pause_time_ms(),

                                cause);

     VMThread::execute(&op);

   } else {

     if (cause == GCCause::_gc_locker

         DEBUG_ONLY(|| cause == GCCause::_scavenge_alot)) {

       // Schedule a standard evacuation pause. We're setting word_size

       // to 0 which means that we are not requesting a post-GC allocation.

       VM_G1IncCollectionPause op(gc_count_before,

                                  0,     /* word_size */

                                  false, /* should_initiate_conc_mark */

                                  g1_policy()->max_pause_time_ms(),

                                  cause);

       VMThread::execute(&op);

     } else {

       // Schedule a Full GC.

       VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);

       VMThread::execute(&op);

     }

   }

 }

 bool G1CollectedHeap::is_in(const void* p) const {

   if (_g1_committed.contains(p)) {

     HeapRegion* hr = _hrs->addr_to_region(p);

     return hr->is_in(p);

   } else {

     return _perm_gen->as_gen()->is_in(p);

   }

 }

 // Iteration functions.

 // Iterates an OopClosure over all ref-containing fields of objects

 // within a HeapRegion.

 class IterateOopClosureRegionClosure: public HeapRegionClosure {

   MemRegion _mr;

   OopClosure* _cl;

 public:

   IterateOopClosureRegionClosure(MemRegion mr, OopClosure* cl)

     : _mr(mr), _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     if (! r->continuesHumongous()) {

       r->oop_iterate(_cl);

     }

     return false;

   }

 };

 void G1CollectedHeap::oop_iterate(OopClosure* cl, bool do_perm) {

   IterateOopClosureRegionClosure blk(_g1_committed, cl);

   _hrs->iterate(&blk);

   if (do_perm) {

     perm_gen()->oop_iterate(cl);

   }

 }

 void G1CollectedHeap::oop_iterate(MemRegion mr, OopClosure* cl, bool do_perm) {

   IterateOopClosureRegionClosure blk(mr, cl);

   _hrs->iterate(&blk);

   if (do_perm) {

     perm_gen()->oop_iterate(cl);

   }

 }

 // Iterates an ObjectClosure over all objects within a HeapRegion.

 class IterateObjectClosureRegionClosure: public HeapRegionClosure {

   ObjectClosure* _cl;

 public:

   IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     if (! r->continuesHumongous()) {

       r->object_iterate(_cl);

     }

     return false;

   }

 };

 void G1CollectedHeap::object_iterate(ObjectClosure* cl, bool do_perm) {

   IterateObjectClosureRegionClosure blk(cl);

   _hrs->iterate(&blk);

   if (do_perm) {

     perm_gen()->object_iterate(cl);

   }

 }

 void G1CollectedHeap::object_iterate_since_last_GC(ObjectClosure* cl) {

   // FIXME: is this right?

   guarantee(false, "object_iterate_since_last_GC not supported by G1 heap");

 }

 // Calls a SpaceClosure on a HeapRegion.

 class SpaceClosureRegionClosure: public HeapRegionClosure {

   SpaceClosure* _cl;

 public:

   SpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     _cl->do_space(r);

     return false;

   }

 };

 void G1CollectedHeap::space_iterate(SpaceClosure* cl) {

   SpaceClosureRegionClosure blk(cl);

   _hrs->iterate(&blk);

 }

 void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) {

   _hrs->iterate(cl);

 }

 void G1CollectedHeap::heap_region_iterate_from(HeapRegion* r,

                                                HeapRegionClosure* cl) {

   _hrs->iterate_from(r, cl);

 }

 void

 G1CollectedHeap::heap_region_iterate_from(int idx, HeapRegionClosure* cl) {

   _hrs->iterate_from(idx, cl);

 }

 HeapRegion* G1CollectedHeap::region_at(size_t idx) { return _hrs->at(idx); }

 void

 G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl,

                                                  int worker,

                                                  jint claim_value) {

   const size_t regions = n_regions();

   const size_t worker_num = (G1CollectedHeap::use_parallel_gc_threads() ? ParallelGCThreads : 1);

   // try to spread out the starting points of the workers

   const size_t start_index = regions / worker_num * (size_t) worker;

   // each worker will actually look at all regions

   for (size_t count = 0; count < regions; ++count) {

     const size_t index = (start_index + count) % regions;

     assert(0 <= index && index < regions, "sanity");

     HeapRegion* r = region_at(index);

     // we'll ignore "continues humongous" regions (we'll process them

     // when we come across their corresponding "start humongous"

     // region) and regions already claimed

     if (r->claim_value() == claim_value || r->continuesHumongous()) {

       continue;

     }

     // OK, try to claim it

     if (r->claimHeapRegion(claim_value)) {

       // success!

       assert(!r->continuesHumongous(), "sanity");

       if (r->startsHumongous()) {

         // If the region is "starts humongous" we'll iterate over its

         // "continues humongous" first; in fact we'll do them

         // first. The order is important. In on case, calling the

         // closure on the "starts humongous" region might de-allocate

         // and clear all its "continues humongous" regions and, as a

         // result, we might end up processing them twice. So, we'll do

         // them first (notice: most closures will ignore them anyway) and

         // then we'll do the "starts humongous" region.

         for (size_t ch_index = index + 1; ch_index < regions; ++ch_index) {

           HeapRegion* chr = region_at(ch_index);

           // if the region has already been claimed or it's not

           // "continues humongous" we're done

           if (chr->claim_value() == claim_value ||

               !chr->continuesHumongous()) {

             break;

           }

           // Noone should have claimed it directly. We can given

           // that we claimed its "starts humongous" region.

           assert(chr->claim_value() != claim_value, "sanity");

           assert(chr->humongous_start_region() == r, "sanity");

           if (chr->claimHeapRegion(claim_value)) {

             // we should always be able to claim it; noone else should

             // be trying to claim this region

             bool res2 = cl->doHeapRegion(chr);

             assert(!res2, "Should not abort");

             // Right now, this holds (i.e., no closure that actually

             // does something with "continues humongous" regions

             // clears them). We might have to weaken it in the future,

             // but let's leave these two asserts here for extra safety.

             assert(chr->continuesHumongous(), "should still be the case");

             assert(chr->humongous_start_region() == r, "sanity");

           } else {

             guarantee(false, "we should not reach here");

           }

         }

       }

       assert(!r->continuesHumongous(), "sanity");

       bool res = cl->doHeapRegion(r);

       assert(!res, "Should not abort");

     }

   }

 }

 class ResetClaimValuesClosure: public HeapRegionClosure {

 public:

   bool doHeapRegion(HeapRegion* r) {

     r->set_claim_value(HeapRegion::InitialClaimValue);

     return false;

   }

 };

 void

 G1CollectedHeap::reset_heap_region_claim_values() {

   ResetClaimValuesClosure blk;

   heap_region_iterate(&blk);

 }

 #ifdef ASSERT

 // This checks whether all regions in the heap have the correct claim

 // value. I also piggy-backed on this a check to ensure that the

 // humongous_start_region() information on "continues humongous"

 // regions is correct.

 class CheckClaimValuesClosure : public HeapRegionClosure {

 private:

   jint _claim_value;

   size_t _failures;

   HeapRegion* _sh_region;

 public:

   CheckClaimValuesClosure(jint claim_value) :

     _claim_value(claim_value), _failures(0), _sh_region(NULL) { }

   bool doHeapRegion(HeapRegion* r) {

     if (r->claim_value() != _claim_value) {

       gclog_or_tty->print_cr("Region ["PTR_FORMAT","PTR_FORMAT"), "

                              "claim value = %d, should be %d",

                              r->bottom(), r->end(), r->claim_value(),

                              _claim_value);

       ++_failures;

     }

     if (!r->isHumongous()) {

       _sh_region = NULL;

     } else if (r->startsHumongous()) {

       _sh_region = r;

     } else if (r->continuesHumongous()) {

       if (r->humongous_start_region() != _sh_region) {

         gclog_or_tty->print_cr("Region ["PTR_FORMAT","PTR_FORMAT"), "

                                "HS = "PTR_FORMAT", should be "PTR_FORMAT,

                                r->bottom(), r->end(),

                                r->humongous_start_region(),

                                _sh_region);

         ++_failures;

       }

     }

     return false;

   }

   size_t failures() {

     return _failures;

   }

 };

 bool G1CollectedHeap::check_heap_region_claim_values(jint claim_value) {

   CheckClaimValuesClosure cl(claim_value);

   heap_region_iterate(&cl);

   return cl.failures() == 0;

 }

 #endif // ASSERT

 void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) {

   HeapRegion* r = g1_policy()->collection_set();

   while (r != NULL) {

     HeapRegion* next = r->next_in_collection_set();

     if (cl->doHeapRegion(r)) {

       cl->incomplete();

       return;

     }

     r = next;

   }

 }

 void G1CollectedHeap::collection_set_iterate_from(HeapRegion* r,

                                                   HeapRegionClosure *cl) {

   if (r == NULL) {

     // The CSet is empty so there's nothing to do.

     return;

   }

   assert(r->in_collection_set(),

          "Start region must be a member of the collection set.");

   HeapRegion* cur = r;

   while (cur != NULL) {

     HeapRegion* next = cur->next_in_collection_set();

     if (cl->doHeapRegion(cur) && false) {

       cl->incomplete();

       return;

     }

     cur = next;

   }

   cur = g1_policy()->collection_set();

   while (cur != r) {

     HeapRegion* next = cur->next_in_collection_set();

     if (cl->doHeapRegion(cur) && false) {

       cl->incomplete();

       return;

     }

     cur = next;

   }

 }

 CompactibleSpace* G1CollectedHeap::first_compactible_space() {

   return _hrs->length() > 0 ? _hrs->at(0) : NULL;

 }

 Space* G1CollectedHeap::space_containing(const void* addr) const {

   Space* res = heap_region_containing(addr);

   if (res == NULL)

     res = perm_gen()->space_containing(addr);

   return res;

 }

 HeapWord* G1CollectedHeap::block_start(const void* addr) const {

   Space* sp = space_containing(addr);

   if (sp != NULL) {

     return sp->block_start(addr);

   }

   return NULL;

 }

 size_t G1CollectedHeap::block_size(const HeapWord* addr) const {

   Space* sp = space_containing(addr);

   assert(sp != NULL, "block_size of address outside of heap");

   return sp->block_size(addr);

 }

 bool G1CollectedHeap::block_is_obj(const HeapWord* addr) const {

   Space* sp = space_containing(addr);

   return sp->block_is_obj(addr);

 }

 bool G1CollectedHeap::supports_tlab_allocation() const {

   return true;

 }

 size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const {

   return HeapRegion::GrainBytes;

 }

 size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const {

   // Return the remaining space in the cur alloc region, but not less than

   // the min TLAB size.

   // Also, this value can be at most the humongous object threshold,

   // since we can't allow tlabs to grow big enough to accomodate

   // humongous objects.

   // We need to store the cur alloc region locally, since it might change

   // between when we test for NULL and when we use it later.

   ContiguousSpace* cur_alloc_space = _cur_alloc_region;

   size_t max_tlab_size = _humongous_object_threshold_in_words * wordSize;

   if (cur_alloc_space == NULL) {

     return max_tlab_size;

   } else {

     return MIN2(MAX2(cur_alloc_space->free(), (size_t)MinTLABSize),

                 max_tlab_size);

   }

 }

 bool G1CollectedHeap::allocs_are_zero_filled() {