Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "incls/_precompiled.incl"

 #include "incls/_g1CollectedHeap.cpp.incl"

 size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0;

 // turn it on so that the contents of the young list (scan-only /

 // to-be-collected) are printed at "strategic" points before / during

 // / after the collection --- this is useful for debugging

 #define YOUNG_LIST_VERBOSE 0

 // CURRENT STATUS

 // This file is under construction.  Search for "FIXME".

 // INVARIANTS/NOTES

 //

 // All allocation activity covered by the G1CollectedHeap interface is

 //   serialized by acquiring the HeapLock.  This happens in

 //   mem_allocate_work, which all such allocation functions call.

 //   (Note that this does not apply to TLAB allocation, which is not part

 //   of this interface: it is done by clients of this interface.)

 // Local to this file.

 class RefineCardTableEntryClosure: public CardTableEntryClosure {

   SuspendibleThreadSet* _sts;

   G1RemSet* _g1rs;

   ConcurrentG1Refine* _cg1r;

   bool _concurrent;

 public:

   RefineCardTableEntryClosure(SuspendibleThreadSet* sts,

                               G1RemSet* g1rs,

                               ConcurrentG1Refine* cg1r) :

     _sts(sts), _g1rs(g1rs), _cg1r(cg1r), _concurrent(true)

   {}

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     bool oops_into_cset = _g1rs->concurrentRefineOneCard(card_ptr, worker_i, false);

     // This path is executed by the concurrent refine or mutator threads,

     // concurrently, and so we do not care if card_ptr contains references

     // that point into the collection set.

     assert(!oops_into_cset, "should be");

     if (_concurrent && _sts->should_yield()) {

       // Caller will actually yield.

       return false;

     }

     // Otherwise, we finished successfully; return true.

     return true;

   }

   void set_concurrent(bool b) { _concurrent = b; }

 };

 class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure {

   int _calls;

   G1CollectedHeap* _g1h;

   CardTableModRefBS* _ctbs;

   int _histo[256];

 public:

   ClearLoggedCardTableEntryClosure() :

     _calls(0)

   {

     _g1h = G1CollectedHeap::heap();

     _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

     for (int i = 0; i < 256; i++) _histo[i] = 0;

   }

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

       _calls++;

       unsigned char* ujb = (unsigned char*)card_ptr;

       int ind = (int)(*ujb);

       _histo[ind]++;

       *card_ptr = -1;

     }

     return true;

   }

   int calls() { return _calls; }

   void print_histo() {

     gclog_or_tty->print_cr("Card table value histogram:");

     for (int i = 0; i < 256; i++) {

       if (_histo[i] != 0) {

         gclog_or_tty->print_cr("  %d: %d", i, _histo[i]);

       }

     }

   }

 };

 class RedirtyLoggedCardTableEntryClosure: public CardTableEntryClosure {

   int _calls;

   G1CollectedHeap* _g1h;

   CardTableModRefBS* _ctbs;

 public:

   RedirtyLoggedCardTableEntryClosure() :

     _calls(0)

   {

     _g1h = G1CollectedHeap::heap();

     _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

   }

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

       _calls++;

       *card_ptr = 0;

     }

     return true;

   }

   int calls() { return _calls; }

 };

 class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure {

 public:

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     *card_ptr = CardTableModRefBS::dirty_card_val();

     return true;

   }

 };

 YoungList::YoungList(G1CollectedHeap* g1h)

   : _g1h(g1h), _head(NULL),

     _length(0),

     _last_sampled_rs_lengths(0),

     _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0)

 {

   guarantee( check_list_empty(false), "just making sure..." );

 }

 void YoungList::push_region(HeapRegion *hr) {

   assert(!hr->is_young(), "should not already be young");

   assert(hr->get_next_young_region() == NULL, "cause it should!");

   hr->set_next_young_region(_head);

   _head = hr;

   hr->set_young();

   double yg_surv_rate = _g1h->g1_policy()->predict_yg_surv_rate((int)_length);

   ++_length;

 }

 void YoungList::add_survivor_region(HeapRegion* hr) {

   assert(hr->is_survivor(), "should be flagged as survivor region");

   assert(hr->get_next_young_region() == NULL, "cause it should!");

   hr->set_next_young_region(_survivor_head);

   if (_survivor_head == NULL) {

     _survivor_tail = hr;

   }

   _survivor_head = hr;

   ++_survivor_length;

 }

 void YoungList::empty_list(HeapRegion* list) {

   while (list != NULL) {

     HeapRegion* next = list->get_next_young_region();

     list->set_next_young_region(NULL);

     list->uninstall_surv_rate_group();

     list->set_not_young();

     list = next;

   }

 }

 void YoungList::empty_list() {

   assert(check_list_well_formed(), "young list should be well formed");

   empty_list(_head);

   _head = NULL;

   _length = 0;

   empty_list(_survivor_head);

   _survivor_head = NULL;

   _survivor_tail = NULL;

   _survivor_length = 0;

   _last_sampled_rs_lengths = 0;

   assert(check_list_empty(false), "just making sure...");

 }

 bool YoungList::check_list_well_formed() {

   bool ret = true;

   size_t length = 0;

   HeapRegion* curr = _head;

   HeapRegion* last = NULL;

   while (curr != NULL) {

     if (!curr->is_young()) {

       gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" "

                              "incorrectly tagged (y: %d, surv: %d)",

                              curr->bottom(), curr->end(),

                              curr->is_young(), curr->is_survivor());

       ret = false;

     }

     ++length;

     last = curr;

     curr = curr->get_next_young_region();

   }

   ret = ret && (length == _length);

   if (!ret) {

     gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!");

     gclog_or_tty->print_cr("###   list has %d entries, _length is %d",

                            length, _length);

   }

   return ret;

 }

 bool YoungList::check_list_empty(bool check_sample) {

   bool ret = true;

   if (_length != 0) {

     gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %d",

                   _length);

     ret = false;

   }

   if (check_sample && _last_sampled_rs_lengths != 0) {

     gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths");

     ret = false;

   }

   if (_head != NULL) {

     gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head");

     ret = false;

   }

   if (!ret) {

     gclog_or_tty->print_cr("### YOUNG LIST does not seem empty");

   }

   return ret;

 }

 void

 YoungList::rs_length_sampling_init() {

   _sampled_rs_lengths = 0;

   _curr               = _head;

 }

 bool

 YoungList::rs_length_sampling_more() {

   return _curr != NULL;

 }

 void

 YoungList::rs_length_sampling_next() {

   assert( _curr != NULL, "invariant" );

   size_t rs_length = _curr->rem_set()->occupied();

   _sampled_rs_lengths += rs_length;

   // The current region may not yet have been added to the

   // incremental collection set (it gets added when it is

   // retired as the current allocation region).

   if (_curr->in_collection_set()) {

     // Update the collection set policy information for this region

     _g1h->g1_policy()->update_incremental_cset_info(_curr, rs_length);

   }

   _curr = _curr->get_next_young_region();

   if (_curr == NULL) {

     _last_sampled_rs_lengths = _sampled_rs_lengths;

     // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths);

   }

 }

 void

 YoungList::reset_auxilary_lists() {

   guarantee( is_empty(), "young list should be empty" );

   assert(check_list_well_formed(), "young list should be well formed");

   // Add survivor regions to SurvRateGroup.

   _g1h->g1_policy()->note_start_adding_survivor_regions();

   _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */);

   for (HeapRegion* curr = _survivor_head;

        curr != NULL;

        curr = curr->get_next_young_region()) {

     _g1h->g1_policy()->set_region_survivors(curr);

     // The region is a non-empty survivor so let's add it to

     // the incremental collection set for the next evacuation

     // pause.

     _g1h->g1_policy()->add_region_to_incremental_cset_rhs(curr);

   }

   _g1h->g1_policy()->note_stop_adding_survivor_regions();

   _head   = _survivor_head;

   _length = _survivor_length;

   if (_survivor_head != NULL) {

     assert(_survivor_tail != NULL, "cause it shouldn't be");

     assert(_survivor_length > 0, "invariant");

     _survivor_tail->set_next_young_region(NULL);

   }

   // Don't clear the survivor list handles until the start of

   // the next evacuation pause - we need it in order to re-tag

   // the survivor regions from this evacuation pause as 'young'

   // at the start of the next.

   _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */);

   assert(check_list_well_formed(), "young list should be well formed");

 }

 void YoungList::print() {

   HeapRegion* lists[] = {_head,   _survivor_head};

   const char* names[] = {"YOUNG", "SURVIVOR"};

   for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) {

     gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]);

     HeapRegion *curr = lists[list];

     if (curr == NULL)

       gclog_or_tty->print_cr("  empty");

     while (curr != NULL) {

       gclog_or_tty->print_cr("  [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, "

                              "age: %4d, y: %d, surv: %d",

                              curr->bottom(), curr->end(),

                              curr->top(),

                              curr->prev_top_at_mark_start(),

                              curr->next_top_at_mark_start(),

                              curr->top_at_conc_mark_count(),

                              curr->age_in_surv_rate_group_cond(),

                              curr->is_young(),

                              curr->is_survivor());

       curr = curr->get_next_young_region();

     }

   }

   gclog_or_tty->print_cr("");

 }

 void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr)

 {

   // Claim the right to put the region on the dirty cards region list

   // by installing a self pointer.

   HeapRegion* next = hr->get_next_dirty_cards_region();

   if (next == NULL) {

     HeapRegion* res = (HeapRegion*)

       Atomic::cmpxchg_ptr(hr, hr->next_dirty_cards_region_addr(),

                           NULL);

     if (res == NULL) {

       HeapRegion* head;

       do {

         // Put the region to the dirty cards region list.

         head = _dirty_cards_region_list;

         next = (HeapRegion*)

           Atomic::cmpxchg_ptr(hr, &_dirty_cards_region_list, head);

         if (next == head) {

           assert(hr->get_next_dirty_cards_region() == hr,

                  "hr->get_next_dirty_cards_region() != hr");

           if (next == NULL) {

             // The last region in the list points to itself.

             hr->set_next_dirty_cards_region(hr);

           } else {

             hr->set_next_dirty_cards_region(next);

           }

         }

       } while (next != head);

     }

   }

 }

 HeapRegion* G1CollectedHeap::pop_dirty_cards_region()

 {

   HeapRegion* head;

   HeapRegion* hr;

   do {

     head = _dirty_cards_region_list;

     if (head == NULL) {

       return NULL;

     }

     HeapRegion* new_head = head->get_next_dirty_cards_region();

     if (head == new_head) {

       // The last region.

       new_head = NULL;

     }

     hr = (HeapRegion*)Atomic::cmpxchg_ptr(new_head, &_dirty_cards_region_list,

                                           head);

   } while (hr != head);

   assert(hr != NULL, "invariant");

   hr->set_next_dirty_cards_region(NULL);

   return hr;

 }

 void G1CollectedHeap::stop_conc_gc_threads() {

   _cg1r->stop();

   _czft->stop();

   _cmThread->stop();

 }

 void G1CollectedHeap::check_ct_logs_at_safepoint() {

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set();

   // Count the dirty cards at the start.

   CountNonCleanMemRegionClosure count1(this);

   ct_bs->mod_card_iterate(&count1);

   int orig_count = count1.n();

   // First clear the logged cards.

   ClearLoggedCardTableEntryClosure clear;

   dcqs.set_closure(&clear);

   dcqs.apply_closure_to_all_completed_buffers();

   dcqs.iterate_closure_all_threads(false);

   clear.print_histo();

   // Now ensure that there's no dirty cards.

   CountNonCleanMemRegionClosure count2(this);

   ct_bs->mod_card_iterate(&count2);

   if (count2.n() != 0) {

     gclog_or_tty->print_cr("Card table has %d entries; %d originally",

                            count2.n(), orig_count);

   }

   guarantee(count2.n() == 0, "Card table should be clean.");

   RedirtyLoggedCardTableEntryClosure redirty;

   JavaThread::dirty_card_queue_set().set_closure(&redirty);

   dcqs.apply_closure_to_all_completed_buffers();

   dcqs.iterate_closure_all_threads(false);

   gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.",

                          clear.calls(), orig_count);

   guarantee(redirty.calls() == clear.calls(),

             "Or else mechanism is broken.");

   CountNonCleanMemRegionClosure count3(this);

   ct_bs->mod_card_iterate(&count3);

   if (count3.n() != orig_count) {

     gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.",

                            orig_count, count3.n());

     guarantee(count3.n() >= orig_count, "Should have restored them all.");

   }

   JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

 }

 // Private class members.

 G1CollectedHeap* G1CollectedHeap::_g1h;

 // Private methods.

 // Finds a HeapRegion that can be used to allocate a given size of block.

 HeapRegion* G1CollectedHeap::newAllocRegion_work(size_t word_size,

                                                  bool do_expand,

                                                  bool zero_filled) {

   ConcurrentZFThread::note_region_alloc();

   HeapRegion* res = alloc_free_region_from_lists(zero_filled);

   if (res == NULL && do_expand) {

     expand(word_size * HeapWordSize);

     res = alloc_free_region_from_lists(zero_filled);

     assert(res == NULL ||

            (!res->isHumongous() &&

             (!zero_filled ||

              res->zero_fill_state() == HeapRegion::Allocated)),

            "Alloc Regions must be zero filled (and non-H)");

   }

   if (res != NULL) {

     if (res->is_empty()) {

       _free_regions--;

     }

     assert(!res->isHumongous() &&

            (!zero_filled || res->zero_fill_state() == HeapRegion::Allocated),

            err_msg("Non-young alloc Regions must be zero filled (and non-H):"

                    " res->isHumongous()=%d, zero_filled=%d, res->zero_fill_state()=%d",

                    res->isHumongous(), zero_filled, res->zero_fill_state()));

     assert(!res->is_on_unclean_list(),

            "Alloc Regions must not be on the unclean list");

     if (G1PrintHeapRegions) {

       gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT", "PTR_FORMAT"], "

                              "top "PTR_FORMAT,

                              res->hrs_index(), res->bottom(), res->end(), res->top());

     }

   }

   return res;

 }

 HeapRegion* G1CollectedHeap::newAllocRegionWithExpansion(int purpose,

                                                          size_t word_size,

                                                          bool zero_filled) {

   HeapRegion* alloc_region = NULL;

   if (_gc_alloc_region_counts[purpose] < g1_policy()->max_regions(purpose)) {

     alloc_region = newAllocRegion_work(word_size, true, zero_filled);

     if (purpose == GCAllocForSurvived && alloc_region != NULL) {

       alloc_region->set_survivor();

     }

     ++_gc_alloc_region_counts[purpose];

   } else {

     g1_policy()->note_alloc_region_limit_reached(purpose);

   }

   return alloc_region;

 }

 // If could fit into free regions w/o expansion, try.

 // Otherwise, if can expand, do so.

 // Otherwise, if using ex regions might help, try with ex given back.

 HeapWord* G1CollectedHeap::humongousObjAllocate(size_t word_size) {

   assert(regions_accounted_for(), "Region leakage!");

   // We can't allocate H regions while cleanupComplete is running, since

   // some of the regions we find to be empty might not yet be added to the

   // unclean list.  (If we're already at a safepoint, this call is

   // unnecessary, not to mention wrong.)

   if (!SafepointSynchronize::is_at_safepoint())

     wait_for_cleanup_complete();

   size_t num_regions =

     round_to(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords;

   // Special case if < one region???

   // Remember the ft size.

   size_t x_size = expansion_regions();

   HeapWord* res = NULL;

   bool eliminated_allocated_from_lists = false;

   // Can the allocation potentially fit in the free regions?

   if (free_regions() >= num_regions) {

     res = _hrs->obj_allocate(word_size);

   }

   if (res == NULL) {

     // Try expansion.

     size_t fs = _hrs->free_suffix();

     if (fs + x_size >= num_regions) {

       expand((num_regions - fs) * HeapRegion::GrainBytes);

       res = _hrs->obj_allocate(word_size);

       assert(res != NULL, "This should have worked.");

     } else {

       // Expansion won't help.  Are there enough free regions if we get rid

       // of reservations?

       size_t avail = free_regions();

       if (avail >= num_regions) {

         res = _hrs->obj_allocate(word_size);

         if (res != NULL) {

           remove_allocated_regions_from_lists();

           eliminated_allocated_from_lists = true;

         }

       }

     }

   }

   if (res != NULL) {

     // Increment by the number of regions allocated.

     // FIXME: Assumes regions all of size GrainBytes.

 #ifndef PRODUCT

     mr_bs()->verify_clean_region(MemRegion(res, res + num_regions *

                                            HeapRegion::GrainWords));

 #endif

     if (!eliminated_allocated_from_lists)

       remove_allocated_regions_from_lists();

     _summary_bytes_used += word_size * HeapWordSize;

     _free_regions -= num_regions;

     _num_humongous_regions += (int) num_regions;

   }

   assert(regions_accounted_for(), "Region Leakage");

   return res;

 }

 HeapWord*

 G1CollectedHeap::attempt_allocation_slow(size_t word_size,

                                          bool permit_collection_pause) {

   HeapWord* res = NULL;

   HeapRegion* allocated_young_region = NULL;

   assert( SafepointSynchronize::is_at_safepoint() ||

           Heap_lock->owned_by_self(), "pre condition of the call" );

   if (isHumongous(word_size)) {

     // Allocation of a humongous object can, in a sense, complete a

     // partial region, if the previous alloc was also humongous, and

     // caused the test below to succeed.

     if (permit_collection_pause)

       do_collection_pause_if_appropriate(word_size);

     res = humongousObjAllocate(word_size);

     assert(_cur_alloc_region == NULL

            || !_cur_alloc_region->isHumongous(),

            "Prevent a regression of this bug.");

   } else {

     // We may have concurrent cleanup working at the time. Wait for it

     // to complete. In the future we would probably want to make the

     // concurrent cleanup truly concurrent by decoupling it from the

     // allocation.

     if (!SafepointSynchronize::is_at_safepoint())

       wait_for_cleanup_complete();

     // If we do a collection pause, this will be reset to a non-NULL

     // value.  If we don't, nulling here ensures that we allocate a new

     // region below.

     if (_cur_alloc_region != NULL) {

       // We're finished with the _cur_alloc_region.

       // As we're builing (at least the young portion) of the collection

       // set incrementally we'll add the current allocation region to

       // the collection set here.

       if (_cur_alloc_region->is_young()) {

         g1_policy()->add_region_to_incremental_cset_lhs(_cur_alloc_region);

       }

       _summary_bytes_used += _cur_alloc_region->used();

       _cur_alloc_region = NULL;

     }

     assert(_cur_alloc_region == NULL, "Invariant.");

     // Completion of a heap region is perhaps a good point at which to do

     // a collection pause.

     if (permit_collection_pause)

       do_collection_pause_if_appropriate(word_size);

     // Make sure we have an allocation region available.

     if (_cur_alloc_region == NULL) {

       if (!SafepointSynchronize::is_at_safepoint())

         wait_for_cleanup_complete();

       bool next_is_young = should_set_young_locked();

       // If the next region is not young, make sure it's zero-filled.

       _cur_alloc_region = newAllocRegion(word_size, !next_is_young);

       if (_cur_alloc_region != NULL) {

         _summary_bytes_used -= _cur_alloc_region->used();

         if (next_is_young) {

           set_region_short_lived_locked(_cur_alloc_region);

           allocated_young_region = _cur_alloc_region;

         }

       }

     }

     assert(_cur_alloc_region == NULL || !_cur_alloc_region->isHumongous(),

            "Prevent a regression of this bug.");

     // Now retry the allocation.

     if (_cur_alloc_region != NULL) {

       if (allocated_young_region != NULL) {

         // We need to ensure that the store to top does not

         // float above the setting of the young type.

         OrderAccess::storestore();

       }

       res = _cur_alloc_region->allocate(word_size);

     }

   }

   // NOTE: fails frequently in PRT

   assert(regions_accounted_for(), "Region leakage!");

   if (res != NULL) {

     if (!SafepointSynchronize::is_at_safepoint()) {

       assert( permit_collection_pause, "invariant" );

       assert( Heap_lock->owned_by_self(), "invariant" );

       Heap_lock->unlock();

     }

     if (allocated_young_region != NULL) {

       HeapRegion* hr = allocated_young_region;

       HeapWord* bottom = hr->bottom();

       HeapWord* end = hr->end();

       MemRegion mr(bottom, end);

       ((CardTableModRefBS*)_g1h->barrier_set())->dirty(mr);

     }

   }

   assert( SafepointSynchronize::is_at_safepoint() ||

           (res == NULL && Heap_lock->owned_by_self()) ||

           (res != NULL && !Heap_lock->owned_by_self()),

           "post condition of the call" );

   return res;

 }

 HeapWord*

 G1CollectedHeap::mem_allocate(size_t word_size,

                               bool   is_noref,

                               bool   is_tlab,

                               bool* gc_overhead_limit_was_exceeded) {

   debug_only(check_for_valid_allocation_state());

   assert(no_gc_in_progress(), "Allocation during gc not allowed");

   HeapWord* result = NULL;

   // Loop until the allocation is satisified,

   // or unsatisfied after GC.

   for (int try_count = 1; /* return or throw */; try_count += 1) {

     int gc_count_before;

     {

       Heap_lock->lock();

       result = attempt_allocation(word_size);

       if (result != NULL) {

         // attempt_allocation should have unlocked the heap lock

         assert(is_in(result), "result not in heap");

         return result;

       }

       // Read the gc count while the heap lock is held.

       gc_count_before = SharedHeap::heap()->total_collections();

       Heap_lock->unlock();

     }

     // Create the garbage collection operation...

     VM_G1CollectForAllocation op(word_size,

                                  gc_count_before);

     // ...and get the VM thread to execute it.

     VMThread::execute(&op);

     if (op.prologue_succeeded()) {

       result = op.result();

       assert(result == NULL || is_in(result), "result not in heap");

       return result;

     }

     // Give a warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::mem_allocate_work retries %d times",

               try_count);

     }

   }

 }

 void G1CollectedHeap::abandon_cur_alloc_region() {

   if (_cur_alloc_region != NULL) {

     // We're finished with the _cur_alloc_region.

     if (_cur_alloc_region->is_empty()) {

       _free_regions++;

       free_region(_cur_alloc_region);

     } else {

       // As we're builing (at least the young portion) of the collection

       // set incrementally we'll add the current allocation region to

       // the collection set here.

       if (_cur_alloc_region->is_young()) {

         g1_policy()->add_region_to_incremental_cset_lhs(_cur_alloc_region);

       }

       _summary_bytes_used += _cur_alloc_region->used();

     }

     _cur_alloc_region = NULL;

   }

 }

 void G1CollectedHeap::abandon_gc_alloc_regions() {

   // first, make sure that the GC alloc region list is empty (it should!)

   assert(_gc_alloc_region_list == NULL, "invariant");

   release_gc_alloc_regions(true /* totally */);

 }

 class PostMCRemSetClearClosure: public HeapRegionClosure {

   ModRefBarrierSet* _mr_bs;

 public:

   PostMCRemSetClearClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}

   bool doHeapRegion(HeapRegion* r) {

     r->reset_gc_time_stamp();

     if (r->continuesHumongous())

       return false;

     HeapRegionRemSet* hrrs = r->rem_set();

     if (hrrs != NULL) hrrs->clear();

     // You might think here that we could clear just the cards

     // corresponding to the used region.  But no: if we leave a dirty card

     // in a region we might allocate into, then it would prevent that card

     // from being enqueued, and cause it to be missed.

     // Re: the performance cost: we shouldn't be doing full GC anyway!

     _mr_bs->clear(MemRegion(r->bottom(), r->end()));

     return false;

   }

 };

 class PostMCRemSetInvalidateClosure: public HeapRegionClosure {

   ModRefBarrierSet* _mr_bs;

 public:

   PostMCRemSetInvalidateClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}

   bool doHeapRegion(HeapRegion* r) {

     if (r->continuesHumongous()) return false;

     if (r->used_region().word_size() != 0) {

       _mr_bs->invalidate(r->used_region(), true /*whole heap*/);

     }

     return false;

   }

 };

 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {

   G1CollectedHeap*   _g1h;

   UpdateRSOopClosure _cl;

   int                _worker_i;

 public:

   RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, int worker_i = 0) :

     _cl(g1->g1_rem_set()->as_HRInto_G1RemSet(), worker_i),

     _worker_i(worker_i),

     _g1h(g1)

   { }

   bool doHeapRegion(HeapRegion* r) {

     if (!r->continuesHumongous()) {

       _cl.set_from(r);

       r->oop_iterate(&_cl);

     }

     return false;

   }

 };

 class ParRebuildRSTask: public AbstractGangTask {

   G1CollectedHeap* _g1;

 public:

   ParRebuildRSTask(G1CollectedHeap* g1)

     : AbstractGangTask("ParRebuildRSTask"),

       _g1(g1)

   { }

   void work(int i) {

     RebuildRSOutOfRegionClosure rebuild_rs(_g1, i);

     _g1->heap_region_par_iterate_chunked(&rebuild_rs, i,

                                          HeapRegion::RebuildRSClaimValue);

   }

 };

 void G1CollectedHeap::do_collection(bool explicit_gc,

                                     bool clear_all_soft_refs,

                                     size_t word_size) {

   if (GC_locker::check_active_before_gc()) {

     return; // GC is disabled (e.g. JNI GetXXXCritical operation)

   }

   ResourceMark rm;

   if (PrintHeapAtGC) {

     Universe::print_heap_before_gc();

   }

   assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");

   assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread");

   const bool do_clear_all_soft_refs = clear_all_soft_refs ||

                            collector_policy()->should_clear_all_soft_refs();

   ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy());

   {

     IsGCActiveMark x;

     // Timing

     bool system_gc = (gc_cause() == GCCause::_java_lang_system_gc);

     assert(!system_gc || explicit_gc, "invariant");

     gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);

     TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);

     TraceTime t(system_gc ? "Full GC (System.gc())" : "Full GC",

                 PrintGC, true, gclog_or_tty);

     TraceMemoryManagerStats tms(true /* fullGC */);

     double start = os::elapsedTime();

     g1_policy()->record_full_collection_start();

     gc_prologue(true);

     increment_total_collections(true /* full gc */);

     size_t g1h_prev_used = used();

     assert(used() == recalculate_used(), "Should be equal");

     if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       prepare_for_verify();

       gclog_or_tty->print(" VerifyBeforeGC:");

       Universe::verify(true);

     }

     assert(regions_accounted_for(), "Region leakage!");

     COMPILER2_PRESENT(DerivedPointerTable::clear());

     // We want to discover references, but not process them yet.

     // This mode is disabled in

     // instanceRefKlass::process_discovered_references if the

     // generation does some collection work, or

     // instanceRefKlass::enqueue_discovered_references if the

     // generation returns without doing any work.

     ref_processor()->disable_discovery();

     ref_processor()->abandon_partial_discovery();

     ref_processor()->verify_no_references_recorded();

     // Abandon current iterations of concurrent marking and concurrent

     // refinement, if any are in progress.

     concurrent_mark()->abort();

     // Make sure we'll choose a new allocation region afterwards.

     abandon_cur_alloc_region();

     abandon_gc_alloc_regions();

     assert(_cur_alloc_region == NULL, "Invariant.");

     g1_rem_set()->as_HRInto_G1RemSet()->cleanupHRRS();

     tear_down_region_lists();

     set_used_regions_to_need_zero_fill();

     // We may have added regions to the current incremental collection

     // set between the last GC or pause and now. We need to clear the

     // incremental collection set and then start rebuilding it afresh

     // after this full GC.

     abandon_collection_set(g1_policy()->inc_cset_head());

     g1_policy()->clear_incremental_cset();

     g1_policy()->stop_incremental_cset_building();

     if (g1_policy()->in_young_gc_mode()) {

       empty_young_list();

       g1_policy()->set_full_young_gcs(true);

     }

     // Temporarily make reference _discovery_ single threaded (non-MT).

     ReferenceProcessorMTMutator rp_disc_ser(ref_processor(), false);

     // Temporarily make refs discovery atomic

     ReferenceProcessorAtomicMutator rp_disc_atomic(ref_processor(), true);

     // Temporarily clear _is_alive_non_header

     ReferenceProcessorIsAliveMutator rp_is_alive_null(ref_processor(), NULL);

     ref_processor()->enable_discovery();

     ref_processor()->setup_policy(do_clear_all_soft_refs);

     // Do collection work

     {

       HandleMark hm;  // Discard invalid handles created during gc

       G1MarkSweep::invoke_at_safepoint(ref_processor(), do_clear_all_soft_refs);

     }

     // Because freeing humongous regions may have added some unclean

     // regions, it is necessary to tear down again before rebuilding.

     tear_down_region_lists();

     rebuild_region_lists();

     _summary_bytes_used = recalculate_used();

     ref_processor()->enqueue_discovered_references();

     COMPILER2_PRESENT(DerivedPointerTable::update_pointers());

     MemoryService::track_memory_usage();

     if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       gclog_or_tty->print(" VerifyAfterGC:");

       prepare_for_verify();

       Universe::verify(false);

     }

     NOT_PRODUCT(ref_processor()->verify_no_references_recorded());

     reset_gc_time_stamp();

     // Since everything potentially moved, we will clear all remembered

     // sets, and clear all cards.  Later we will rebuild remebered

     // sets. We will also reset the GC time stamps of the regions.

     PostMCRemSetClearClosure rs_clear(mr_bs());

     heap_region_iterate(&rs_clear);

     // Resize the heap if necessary.

     resize_if_necessary_after_full_collection(explicit_gc ? 0 : word_size);

     if (_cg1r->use_cache()) {

       _cg1r->clear_and_record_card_counts();

       _cg1r->clear_hot_cache();

     }

     // Rebuild remembered sets of all regions.

     if (G1CollectedHeap::use_parallel_gc_threads()) {

       ParRebuildRSTask rebuild_rs_task(this);

       assert(check_heap_region_claim_values(

              HeapRegion::InitialClaimValue), "sanity check");

       set_par_threads(workers()->total_workers());

       workers()->run_task(&rebuild_rs_task);

       set_par_threads(0);

       assert(check_heap_region_claim_values(

              HeapRegion::RebuildRSClaimValue), "sanity check");

       reset_heap_region_claim_values();

     } else {

       RebuildRSOutOfRegionClosure rebuild_rs(this);

       heap_region_iterate(&rebuild_rs);

     }

     if (PrintGC) {

       print_size_transition(gclog_or_tty, g1h_prev_used, used(), capacity());

     }

     if (true) { // FIXME

       // Ask the permanent generation to adjust size for full collections

       perm()->compute_new_size();

     }

     // Start a new incremental collection set for the next pause

     assert(g1_policy()->collection_set() == NULL, "must be");

     g1_policy()->start_incremental_cset_building();

     // Clear the _cset_fast_test bitmap in anticipation of adding

     // regions to the incremental collection set for the next

     // evacuation pause.

     clear_cset_fast_test();

     double end = os::elapsedTime();

     g1_policy()->record_full_collection_end();

 #ifdef TRACESPINNING

     ParallelTaskTerminator::print_termination_counts();

 #endif

     gc_epilogue(true);

     // Discard all rset updates

     JavaThread::dirty_card_queue_set().abandon_logs();

     assert(!G1DeferredRSUpdate

            || (G1DeferredRSUpdate && (dirty_card_queue_set().completed_buffers_num() == 0)), "Should not be any");

     assert(regions_accounted_for(), "Region leakage!");

   }

   if (g1_policy()->in_young_gc_mode()) {

     _young_list->reset_sampled_info();

     // At this point there should be no regions in the

     // entire heap tagged as young.

     assert( check_young_list_empty(true /* check_heap */),

             "young list should be empty at this point");

   }

   // Update the number of full collections that have been completed.

   increment_full_collections_completed(false /* outer */);

   if (PrintHeapAtGC) {

     Universe::print_heap_after_gc();

   }

 }

 void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {

   do_collection(true,                /* explicit_gc */

                 clear_all_soft_refs,

                 0                    /* word_size */);

 }

 // This code is mostly copied from TenuredGeneration.

 void

 G1CollectedHeap::

 resize_if_necessary_after_full_collection(size_t word_size) {

   assert(MinHeapFreeRatio <= MaxHeapFreeRatio, "sanity check");

   // Include the current allocation, if any, and bytes that will be

   // pre-allocated to support collections, as "used".

   const size_t used_after_gc = used();

   const size_t capacity_after_gc = capacity();

   const size_t free_after_gc = capacity_after_gc - used_after_gc;

   // This is enforced in arguments.cpp.

   assert(MinHeapFreeRatio <= MaxHeapFreeRatio,

          "otherwise the code below doesn't make sense");

   // We don't have floating point command-line arguments

   const double minimum_free_percentage = (double) MinHeapFreeRatio / 100.0;

   const double maximum_used_percentage = 1.0 - minimum_free_percentage;

   const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100.0;

   const double minimum_used_percentage = 1.0 - maximum_free_percentage;

   const size_t min_heap_size = collector_policy()->min_heap_byte_size();

   const size_t max_heap_size = collector_policy()->max_heap_byte_size();

   // We have to be careful here as these two calculations can overflow

   // 32-bit size_t's.

   double used_after_gc_d = (double) used_after_gc;

   double minimum_desired_capacity_d = used_after_gc_d / maximum_used_percentage;

   double maximum_desired_capacity_d = used_after_gc_d / minimum_used_percentage;

   // Let's make sure that they are both under the max heap size, which

   // by default will make them fit into a size_t.

   double desired_capacity_upper_bound = (double) max_heap_size;

   minimum_desired_capacity_d = MIN2(minimum_desired_capacity_d,

                                     desired_capacity_upper_bound);

   maximum_desired_capacity_d = MIN2(maximum_desired_capacity_d,

                                     desired_capacity_upper_bound);

   // We can now safely turn them into size_t's.

   size_t minimum_desired_capacity = (size_t) minimum_desired_capacity_d;

   size_t maximum_desired_capacity = (size_t) maximum_desired_capacity_d;

   // This assert only makes sense here, before we adjust them

   // with respect to the min and max heap size.

   assert(minimum_desired_capacity <= maximum_desired_capacity,

          err_msg("minimum_desired_capacity = "SIZE_FORMAT", "

                  "maximum_desired_capacity = "SIZE_FORMAT,

                  minimum_desired_capacity, maximum_desired_capacity));

   // Should not be greater than the heap max size. No need to adjust

   // it with respect to the heap min size as it's a lower bound (i.e.,

   // we'll try to make the capacity larger than it, not smaller).

   minimum_desired_capacity = MIN2(minimum_desired_capacity, max_heap_size);

   // Should not be less than the heap min size. No need to adjust it

   // with respect to the heap max size as it's an upper bound (i.e.,

   // we'll try to make the capacity smaller than it, not greater).

   maximum_desired_capacity =  MAX2(maximum_desired_capacity, min_heap_size);

   if (PrintGC && Verbose) {

     const double free_percentage =

       (double) free_after_gc / (double) capacity_after_gc;

     gclog_or_tty->print_cr("Computing new size after full GC ");

     gclog_or_tty->print_cr("  "

                            "  minimum_free_percentage: %6.2f",

                            minimum_free_percentage);

     gclog_or_tty->print_cr("  "

                            "  maximum_free_percentage: %6.2f",

                            maximum_free_percentage);

     gclog_or_tty->print_cr("  "

                            "  capacity: %6.1fK"

                            "  minimum_desired_capacity: %6.1fK"

                            "  maximum_desired_capacity: %6.1fK",

                            (double) capacity_after_gc / (double) K,

                            (double) minimum_desired_capacity / (double) K,

                            (double) maximum_desired_capacity / (double) K);

     gclog_or_tty->print_cr("  "

                            "  free_after_gc: %6.1fK"

                            "  used_after_gc: %6.1fK",

                            (double) free_after_gc / (double) K,

                            (double) used_after_gc / (double) K);

     gclog_or_tty->print_cr("  "

                            "   free_percentage: %6.2f",

                            free_percentage);

   }

   if (capacity_after_gc < minimum_desired_capacity) {

     // Don't expand unless it's significant

     size_t expand_bytes = minimum_desired_capacity - capacity_after_gc;

     expand(expand_bytes);

     if (PrintGC && Verbose) {

       gclog_or_tty->print_cr("  "

                              "  expanding:"

                              "  max_heap_size: %6.1fK"

                              "  minimum_desired_capacity: %6.1fK"

                              "  expand_bytes: %6.1fK",

                              (double) max_heap_size / (double) K,

                              (double) minimum_desired_capacity / (double) K,

                              (double) expand_bytes / (double) K);

     }

     // No expansion, now see if we want to shrink

   } else if (capacity_after_gc > maximum_desired_capacity) {

     // Capacity too large, compute shrinking size

     size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity;

     shrink(shrink_bytes);

     if (PrintGC && Verbose) {

       gclog_or_tty->print_cr("  "

                              "  shrinking:"

                              "  min_heap_size: %6.1fK"

                              "  maximum_desired_capacity: %6.1fK"

                              "  shrink_bytes: %6.1fK",

                              (double) min_heap_size / (double) K,

                              (double) maximum_desired_capacity / (double) K,

                              (double) shrink_bytes / (double) K);

     }

   }

 }

 HeapWord*

 G1CollectedHeap::satisfy_failed_allocation(size_t word_size) {

   HeapWord* result = NULL;

   // In a G1 heap, we're supposed to keep allocation from failing by

   // incremental pauses.  Therefore, at least for now, we'll favor

   // expansion over collection.  (This might change in the future if we can

   // do something smarter than full collection to satisfy a failed alloc.)

   result = expand_and_allocate(word_size);

   if (result != NULL) {

     assert(is_in(result), "result not in heap");

     return result;

   }

   // OK, I guess we have to try collection.

   do_collection(false, false, word_size);

   result = attempt_allocation(word_size, /*permit_collection_pause*/false);

   if (result != NULL) {

     assert(is_in(result), "result not in heap");

     return result;

   }

   // Try collecting soft references.

   do_collection(false, true, word_size);

   result = attempt_allocation(word_size, /*permit_collection_pause*/false);

   if (result != NULL) {

     assert(is_in(result), "result not in heap");

     return result;

   }

   assert(!collector_policy()->should_clear_all_soft_refs(),

     "Flag should have been handled and cleared prior to this point");

   // What else?  We might try synchronous finalization later.  If the total

   // space available is large enough for the allocation, then a more

   // complete compaction phase than we've tried so far might be

   // appropriate.

   return NULL;

 }

 // Attempting to expand the heap sufficiently

 // to support an allocation of the given "word_size".  If

 // successful, perform the allocation and return the address of the

 // allocated block, or else "NULL".

 HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size) {

   size_t expand_bytes = word_size * HeapWordSize;

   if (expand_bytes < MinHeapDeltaBytes) {

     expand_bytes = MinHeapDeltaBytes;

   }

   expand(expand_bytes);

   assert(regions_accounted_for(), "Region leakage!");

   HeapWord* result = attempt_allocation(word_size, false /* permit_collection_pause */);

   return result;

 }

 size_t G1CollectedHeap::free_region_if_totally_empty(HeapRegion* hr) {

   size_t pre_used = 0;

   size_t cleared_h_regions = 0;

   size_t freed_regions = 0;

   UncleanRegionList local_list;

   free_region_if_totally_empty_work(hr, pre_used, cleared_h_regions,

                                     freed_regions, &local_list);

   finish_free_region_work(pre_used, cleared_h_regions, freed_regions,

                           &local_list);

   return pre_used;

 }

 void

 G1CollectedHeap::free_region_if_totally_empty_work(HeapRegion* hr,

                                                    size_t& pre_used,

                                                    size_t& cleared_h,

                                                    size_t& freed_regions,

                                                    UncleanRegionList* list,

                                                    bool par) {

   assert(!hr->continuesHumongous(), "should have filtered these out");

   size_t res = 0;

   if (hr->used() > 0 && hr->garbage_bytes() == hr->used() &&

       !hr->is_young()) {

     if (G1PolicyVerbose > 0)

       gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)"

                                                                                " during cleanup", hr, hr->used());

     free_region_work(hr, pre_used, cleared_h, freed_regions, list, par);

   }

 }

 // FIXME: both this and shrink could probably be more efficient by

 // doing one "VirtualSpace::expand_by" call rather than several.

 void G1CollectedHeap::expand(size_t expand_bytes) {

   size_t old_mem_size = _g1_storage.committed_size();

   // We expand by a minimum of 1K.

   expand_bytes = MAX2(expand_bytes, (size_t)K);

   size_t aligned_expand_bytes =

     ReservedSpace::page_align_size_up(expand_bytes);

   aligned_expand_bytes = align_size_up(aligned_expand_bytes,

                                        HeapRegion::GrainBytes);

   expand_bytes = aligned_expand_bytes;

   while (expand_bytes > 0) {

     HeapWord* base = (HeapWord*)_g1_storage.high();

     // Commit more storage.

     bool successful = _g1_storage.expand_by(HeapRegion::GrainBytes);

     if (!successful) {

         expand_bytes = 0;

     } else {

       expand_bytes -= HeapRegion::GrainBytes;

       // Expand the committed region.

       HeapWord* high = (HeapWord*) _g1_storage.high();

       _g1_committed.set_end(high);

       // Create a new HeapRegion.

       MemRegion mr(base, high);

       bool is_zeroed = !_g1_max_committed.contains(base);

       HeapRegion* hr = new HeapRegion(_bot_shared, mr, is_zeroed);

       // Now update max_committed if necessary.

       _g1_max_committed.set_end(MAX2(_g1_max_committed.end(), high));

       // Add it to the HeapRegionSeq.

       _hrs->insert(hr);

       // Set the zero-fill state, according to whether it's already

       // zeroed.

       {

         MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);

         if (is_zeroed) {

           hr->set_zero_fill_complete();

           put_free_region_on_list_locked(hr);

         } else {

           hr->set_zero_fill_needed();

           put_region_on_unclean_list_locked(hr);

         }

       }

       _free_regions++;

       // And we used up an expansion region to create it.

       _expansion_regions--;

       // Tell the cardtable about it.

       Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);

       // And the offset table as well.

       _bot_shared->resize(_g1_committed.word_size());

     }

   }

   if (Verbose && PrintGC) {

     size_t new_mem_size = _g1_storage.committed_size();

     gclog_or_tty->print_cr("Expanding garbage-first heap from %ldK by %ldK to %ldK",

                            old_mem_size/K, aligned_expand_bytes/K,

                            new_mem_size/K);

   }

 }

 void G1CollectedHeap::shrink_helper(size_t shrink_bytes)

 {

   size_t old_mem_size = _g1_storage.committed_size();

   size_t aligned_shrink_bytes =

     ReservedSpace::page_align_size_down(shrink_bytes);

   aligned_shrink_bytes = align_size_down(aligned_shrink_bytes,

                                          HeapRegion::GrainBytes);

   size_t num_regions_deleted = 0;

   MemRegion mr = _hrs->shrink_by(aligned_shrink_bytes, num_regions_deleted);

   assert(mr.end() == (HeapWord*)_g1_storage.high(), "Bad shrink!");

   if (mr.byte_size() > 0)

     _g1_storage.shrink_by(mr.byte_size());

   assert(mr.start() == (HeapWord*)_g1_storage.high(), "Bad shrink!");

   _g1_committed.set_end(mr.start());

   _free_regions -= num_regions_deleted;

   _expansion_regions += num_regions_deleted;

   // Tell the cardtable about it.

   Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);

   // And the offset table as well.

   _bot_shared->resize(_g1_committed.word_size());

   HeapRegionRemSet::shrink_heap(n_regions());

   if (Verbose && PrintGC) {

     size_t new_mem_size = _g1_storage.committed_size();

     gclog_or_tty->print_cr("Shrinking garbage-first heap from %ldK by %ldK to %ldK",

                            old_mem_size/K, aligned_shrink_bytes/K,

                            new_mem_size/K);

   }

 }

 void G1CollectedHeap::shrink(size_t shrink_bytes) {

   release_gc_alloc_regions(true /* totally */);

   tear_down_region_lists();  // We will rebuild them in a moment.

   shrink_helper(shrink_bytes);

   rebuild_region_lists();

 }

 // Public methods.

 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away

 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list

 #endif // _MSC_VER

 G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) :

   SharedHeap(policy_),

   _g1_policy(policy_),

   _dirty_card_queue_set(false),

   _into_cset_dirty_card_queue_set(false),

   _ref_processor(NULL),

   _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)),

   _bot_shared(NULL),

   _par_alloc_during_gc_lock(Mutex::leaf, "par alloc during GC lock"),

   _objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL),

   _evac_failure_scan_stack(NULL) ,

   _mark_in_progress(false),

   _cg1r(NULL), _czft(NULL), _summary_bytes_used(0),

   _cur_alloc_region(NULL),

   _refine_cte_cl(NULL),

   _free_region_list(NULL), _free_region_list_size(0),

   _free_regions(0),

   _full_collection(false),

   _unclean_region_list(),

   _unclean_regions_coming(false),

   _young_list(new YoungList(this)),

   _gc_time_stamp(0),

   _surviving_young_words(NULL),

   _full_collections_completed(0),

   _in_cset_fast_test(NULL),

   _in_cset_fast_test_base(NULL),

   _dirty_cards_region_list(NULL) {

   _g1h = this; // To catch bugs.

   if (_process_strong_tasks == NULL || !_process_strong_tasks->valid()) {

     vm_exit_during_initialization("Failed necessary allocation.");

   }

   _humongous_object_threshold_in_words = HeapRegion::GrainWords / 2;

   int n_queues = MAX2((int)ParallelGCThreads, 1);

   _task_queues = new RefToScanQueueSet(n_queues);

   int n_rem_sets = HeapRegionRemSet::num_par_rem_sets();

   assert(n_rem_sets > 0, "Invariant.");

   HeapRegionRemSetIterator** iter_arr =

     NEW_C_HEAP_ARRAY(HeapRegionRemSetIterator*, n_queues);

   for (int i = 0; i < n_queues; i++) {

     iter_arr[i] = new HeapRegionRemSetIterator();

   }

   _rem_set_iterator = iter_arr;

   for (int i = 0; i < n_queues; i++) {

     RefToScanQueue* q = new RefToScanQueue();

     q->initialize();

     _task_queues->register_queue(i, q);

   }

   for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {

     _gc_alloc_regions[ap]          = NULL;

     _gc_alloc_region_counts[ap]    = 0;

     _retained_gc_alloc_regions[ap] = NULL;

     // by default, we do not retain a GC alloc region for each ap;

     // we'll override this, when appropriate, below

     _retain_gc_alloc_region[ap]    = false;

   }

   // We will try to remember the last half-full tenured region we

   // allocated to at the end of a collection so that we can re-use it

   // during the next collection.

   _retain_gc_alloc_region[GCAllocForTenured]  = true;

   guarantee(_task_queues != NULL, "task_queues allocation failure.");

 }

 jint G1CollectedHeap::initialize() {

   CollectedHeap::pre_initialize();

   os::enable_vtime();

   // Necessary to satisfy locking discipline assertions.

   MutexLocker x(Heap_lock);

   // While there are no constraints in the GC code that HeapWordSize

   // be any particular value, there are multiple other areas in the

   // system which believe this to be true (e.g. oop->object_size in some

   // cases incorrectly returns the size in wordSize units rather than

   // HeapWordSize).

   guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize");

   size_t init_byte_size = collector_policy()->initial_heap_byte_size();

   size_t max_byte_size = collector_policy()->max_heap_byte_size();

   // Ensure that the sizes are properly aligned.

   Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap");

   Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap");

   _cg1r = new ConcurrentG1Refine();

   // Reserve the maximum.

   PermanentGenerationSpec* pgs = collector_policy()->permanent_generation();

   // Includes the perm-gen.

   const size_t total_reserved = max_byte_size + pgs->max_size();

   char* addr = Universe::preferred_heap_base(total_reserved, Universe::UnscaledNarrowOop);

   ReservedSpace heap_rs(max_byte_size + pgs->max_size(),

                         HeapRegion::GrainBytes,

                         false /*ism*/, addr);

   if (UseCompressedOops) {

     if (addr != NULL && !heap_rs.is_reserved()) {

       // Failed to reserve at specified address - the requested memory

       // region is taken already, for example, by 'java' launcher.

       // Try again to reserver heap higher.

       addr = Universe::preferred_heap_base(total_reserved, Universe::ZeroBasedNarrowOop);

       ReservedSpace heap_rs0(total_reserved, HeapRegion::GrainBytes,

                              false /*ism*/, addr);

       if (addr != NULL && !heap_rs0.is_reserved()) {

         // Failed to reserve at specified address again - give up.

         addr = Universe::preferred_heap_base(total_reserved, Universe::HeapBasedNarrowOop);

         assert(addr == NULL, "");

         ReservedSpace heap_rs1(total_reserved, HeapRegion::GrainBytes,

                                false /*ism*/, addr);

         heap_rs = heap_rs1;

       } else {

         heap_rs = heap_rs0;

       }

     }

   }

   if (!heap_rs.is_reserved()) {

     vm_exit_during_initialization("Could not reserve enough space for object heap");

     return JNI_ENOMEM;

   }

   // It is important to do this in a way such that concurrent readers can't

   // temporarily think somethings in the heap.  (I've actually seen this

   // happen in asserts: DLD.)

   _reserved.set_word_size(0);

   _reserved.set_start((HeapWord*)heap_rs.base());

   _reserved.set_end((HeapWord*)(heap_rs.base() + heap_rs.size()));

   _expansion_regions = max_byte_size/HeapRegion::GrainBytes;

   _num_humongous_regions = 0;

   // Create the gen rem set (and barrier set) for the entire reserved region.

   _rem_set = collector_policy()->create_rem_set(_reserved, 2);

   set_barrier_set(rem_set()->bs());

   if (barrier_set()->is_a(BarrierSet::ModRef)) {

     _mr_bs = (ModRefBarrierSet*)_barrier_set;

   } else {

     vm_exit_during_initialization("G1 requires a mod ref bs.");

     return JNI_ENOMEM;

   }

   // Also create a G1 rem set.

   if (G1UseHRIntoRS) {

     if (mr_bs()->is_a(BarrierSet::CardTableModRef)) {

       _g1_rem_set = new HRInto_G1RemSet(this, (CardTableModRefBS*)mr_bs());

     } else {

       vm_exit_during_initialization("G1 requires a cardtable mod ref bs.");

       return JNI_ENOMEM;

     }

   } else {

     _g1_rem_set = new StupidG1RemSet(this);

   }

   // Carve out the G1 part of the heap.

   ReservedSpace g1_rs   = heap_rs.first_part(max_byte_size);

   _g1_reserved = MemRegion((HeapWord*)g1_rs.base(),

                            g1_rs.size()/HeapWordSize);

   ReservedSpace perm_gen_rs = heap_rs.last_part(max_byte_size);

   _perm_gen = pgs->init(perm_gen_rs, pgs->init_size(), rem_set());

   _g1_storage.initialize(g1_rs, 0);

   _g1_committed = MemRegion((HeapWord*)_g1_storage.low(), (size_t) 0);

   _g1_max_committed = _g1_committed;

   _hrs = new HeapRegionSeq(_expansion_regions);

   guarantee(_hrs != NULL, "Couldn't allocate HeapRegionSeq");

   guarantee(_cur_alloc_region == NULL, "from constructor");

   // 6843694 - ensure that the maximum region index can fit

   // in the remembered set structures.

   const size_t max_region_idx = ((size_t)1 << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1;

   guarantee((max_regions() - 1) <= max_region_idx, "too many regions");

   size_t max_cards_per_region = ((size_t)1 << (sizeof(CardIdx_t)*BitsPerByte-1)) - 1;

   guarantee(HeapRegion::CardsPerRegion > 0, "make sure it's initialized");

   guarantee((size_t) HeapRegion::CardsPerRegion < max_cards_per_region,

             "too many cards per region");

   _bot_shared = new G1BlockOffsetSharedArray(_reserved,

                                              heap_word_size(init_byte_size));

   _g1h = this;

    _in_cset_fast_test_length = max_regions();

    _in_cset_fast_test_base = NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);

    // We're biasing _in_cset_fast_test to avoid subtracting the

    // beginning of the heap every time we want to index; basically

    // it's the same with what we do with the card table.

    _in_cset_fast_test = _in_cset_fast_test_base -

                 ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);

    // Clear the _cset_fast_test bitmap in anticipation of adding

    // regions to the incremental collection set for the first

    // evacuation pause.

    clear_cset_fast_test();

   // Create the ConcurrentMark data structure and thread.

   // (Must do this late, so that "max_regions" is defined.)

   _cm       = new ConcurrentMark(heap_rs, (int) max_regions());

   _cmThread = _cm->cmThread();

   // ...and the concurrent zero-fill thread, if necessary.

   if (G1ConcZeroFill) {

     _czft = new ConcurrentZFThread();

   }

   // Initialize the from_card cache structure of HeapRegionRemSet.

   HeapRegionRemSet::init_heap(max_regions());

   // Now expand into the initial heap size.

   expand(init_byte_size);

   // Perform any initialization actions delegated to the policy.

   g1_policy()->init();

   g1_policy()->note_start_of_mark_thread();

   _refine_cte_cl =

     new RefineCardTableEntryClosure(ConcurrentG1RefineThread::sts(),

                                     g1_rem_set(),

                                     concurrent_g1_refine());

   JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

   JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,

                                                SATB_Q_FL_lock,

                                                G1SATBProcessCompletedThreshold,

                                                Shared_SATB_Q_lock);

   JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,

                                                 DirtyCardQ_FL_lock,

                                                 concurrent_g1_refine()->yellow_zone(),

                                                 concurrent_g1_refine()->red_zone(),

                                                 Shared_DirtyCardQ_lock);

   if (G1DeferredRSUpdate) {

     dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,

                                       DirtyCardQ_FL_lock,

                                       -1, // never trigger processing

                                       -1, // no limit on length

                                       Shared_DirtyCardQ_lock,

                                       &JavaThread::dirty_card_queue_set());

   }

   // Initialize the card queue set used to hold cards containing

   // references into the collection set.

   _into_cset_dirty_card_queue_set.initialize(DirtyCardQ_CBL_mon,

                                              DirtyCardQ_FL_lock,

                                              -1, // never trigger processing

                                              -1, // no limit on length

                                              Shared_DirtyCardQ_lock,

                                              &JavaThread::dirty_card_queue_set());

   // In case we're keeping closure specialization stats, initialize those

   // counts and that mechanism.

   SpecializationStats::clear();

   _gc_alloc_region_list = NULL;

   // Do later initialization work for concurrent refinement.

   _cg1r->init();

   return JNI_OK;

 }

 void G1CollectedHeap::ref_processing_init() {

   SharedHeap::ref_processing_init();

   MemRegion mr = reserved_region();

   _ref_processor = ReferenceProcessor::create_ref_processor(

                                          mr,    // span

                                          false, // Reference discovery is not atomic

                                                 // (though it shouldn't matter here.)

                                          true,  // mt_discovery

                                          NULL,  // is alive closure: need to fill this in for efficiency

                                          ParallelGCThreads,

                                          ParallelRefProcEnabled,

                                          true); // Setting next fields of discovered

                                                 // lists requires a barrier.

 }

 size_t G1CollectedHeap::capacity() const {

   return _g1_committed.byte_size();

 }

 void G1CollectedHeap::iterate_dirty_card_closure(CardTableEntryClosure* cl,

                                                  DirtyCardQueue* into_cset_dcq,

                                                  bool concurrent,

                                                  int worker_i) {

   // Clean cards in the hot card cache

   concurrent_g1_refine()->clean_up_cache(worker_i, g1_rem_set(), into_cset_dcq);

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   int n_completed_buffers = 0;

   while (dcqs.apply_closure_to_completed_buffer(cl, worker_i, 0, true)) {

     n_completed_buffers++;

   }

   g1_policy()->record_update_rs_processed_buffers(worker_i,

                                                   (double) n_completed_buffers);

   dcqs.clear_n_completed_buffers();

   assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!");

 }

 // Computes the sum of the storage used by the various regions.

 size_t G1CollectedHeap::used() const {

   assert(Heap_lock->owner() != NULL,

          "Should be owned on this thread's behalf.");

   size_t result = _summary_bytes_used;

   // Read only once in case it is set to NULL concurrently

   HeapRegion* hr = _cur_alloc_region;

   if (hr != NULL)

     result += hr->used();

   return result;

 }

 size_t G1CollectedHeap::used_unlocked() const {

   size_t result = _summary_bytes_used;

   return result;

 }

 class SumUsedClosure: public HeapRegionClosure {

   size_t _used;

 public:

   SumUsedClosure() : _used(0) {}

   bool doHeapRegion(HeapRegion* r) {

     if (!r->continuesHumongous()) {

       _used += r->used();

     }

     return false;

   }

   size_t result() { return _used; }

 };

 size_t G1CollectedHeap::recalculate_used() const {

   SumUsedClosure blk;

   _hrs->iterate(&blk);

   return blk.result();

 }

 #ifndef PRODUCT

 class SumUsedRegionsClosure: public HeapRegionClosure {

   size_t _num;

 public:

   SumUsedRegionsClosure() : _num(0) {}

   bool doHeapRegion(HeapRegion* r) {

     if (r->continuesHumongous() || r->used() > 0 || r->is_gc_alloc_region()) {

       _num += 1;

     }

     return false;

   }

   size_t result() { return _num; }

 };

 size_t G1CollectedHeap::recalculate_used_regions() const {

   SumUsedRegionsClosure blk;

   _hrs->iterate(&blk);

   return blk.result();

 }

 #endif // PRODUCT

 size_t G1CollectedHeap::unsafe_max_alloc() {

   if (_free_regions > 0) return HeapRegion::GrainBytes;

   // otherwise, is there space in the current allocation region?

   // We need to store the current allocation region in a local variable

   // here. The problem is that this method doesn't take any locks and

   // there may be other threads which overwrite the current allocation

   // region field. attempt_allocation(), for example, sets it to NULL

   // and this can happen *after* the NULL check here but before the call

   // to free(), resulting in a SIGSEGV. Note that this doesn't appear

   // to be a problem in the optimized build, since the two loads of the

   // current allocation region field are optimized away.

   HeapRegion* car = _cur_alloc_region;

   // FIXME: should iterate over all regions?

   if (car == NULL) {

     return 0;

   }

   return car->free();

 }

 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {

   return

     ((cause == GCCause::_gc_locker           && GCLockerInvokesConcurrent) ||

      (cause == GCCause::_java_lang_system_gc && ExplicitGCInvokesConcurrent));

 }

 void G1CollectedHeap::increment_full_collections_completed(bool outer) {

   MonitorLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag);

   // We have already incremented _total_full_collections at the start

   // of the GC, so total_full_collections() represents how many full

   // collections have been started.

   unsigned int full_collections_started = total_full_collections();

   // Given that this method is called at the end of a Full GC or of a

   // concurrent cycle, and those can be nested (i.e., a Full GC can

   // interrupt a concurrent cycle), the number of full collections

   // completed should be either one (in the case where there was no

   // nesting) or two (when a Full GC interrupted a concurrent cycle)

   // behind the number of full collections started.

   // This is the case for the inner caller, i.e. a Full GC.

   assert(outer ||

          (full_collections_started == _full_collections_completed + 1) ||

          (full_collections_started == _full_collections_completed + 2),

          err_msg("for inner caller: full_collections_started = %u "

                  "is inconsistent with _full_collections_completed = %u",

                  full_collections_started, _full_collections_completed));

   // This is the case for the outer caller, i.e. the concurrent cycle.

   assert(!outer ||

          (full_collections_started == _full_collections_completed + 1),

          err_msg("for outer caller: full_collections_started = %u "

                  "is inconsistent with _full_collections_completed = %u",

                  full_collections_started, _full_collections_completed));

   _full_collections_completed += 1;

   // We need to clear the "in_progress" flag in the CM thread before

   // we wake up any waiters (especially when ExplicitInvokesConcurrent

   // is set) so that if a waiter requests another System.gc() it doesn't

   // incorrectly see that a marking cyle is still in progress.

   if (outer) {

     _cmThread->clear_in_progress();

   }

   // This notify_all() will ensure that a thread that called

   // System.gc() with (with ExplicitGCInvokesConcurrent set or not)

   // and it's waiting for a full GC to finish will be woken up. It is

   // waiting in VM_G1IncCollectionPause::doit_epilogue().

   FullGCCount_lock->notify_all();

 }

 void G1CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) {

   assert(Thread::current()->is_VM_thread(), "Precondition#1");

   assert(Heap_lock->is_locked(), "Precondition#2");

   GCCauseSetter gcs(this, cause);

   switch (cause) {

     case GCCause::_heap_inspection:

     case GCCause::_heap_dump: {

       HandleMark hm;

       do_full_collection(false);         // don't clear all soft refs

       break;

     }

     default: // XXX FIX ME

       ShouldNotReachHere(); // Unexpected use of this function

   }

 }

 void G1CollectedHeap::collect(GCCause::Cause cause) {

   // The caller doesn't have the Heap_lock

   assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock");

   unsigned int gc_count_before;

   unsigned int full_gc_count_before;

   {

     MutexLocker ml(Heap_lock);

     // Read the GC count while holding the Heap_lock

     gc_count_before = SharedHeap::heap()->total_collections();

     full_gc_count_before = SharedHeap::heap()->total_full_collections();

     // Don't want to do a GC until cleanup is completed.

     wait_for_cleanup_complete();

     // We give up heap lock; VMThread::execute gets it back below

   }

   if (should_do_concurrent_full_gc(cause)) {

     // Schedule an initial-mark evacuation pause that will start a

     // concurrent cycle.

     VM_G1IncCollectionPause op(gc_count_before,

                                true, /* should_initiate_conc_mark */

                                g1_policy()->max_pause_time_ms(),

                                cause);

     VMThread::execute(&op);

   } else {

     if (cause == GCCause::_gc_locker

         DEBUG_ONLY(|| cause == GCCause::_scavenge_alot)) {

       // Schedule a standard evacuation pause.

       VM_G1IncCollectionPause op(gc_count_before,

                                  false, /* should_initiate_conc_mark */

                                  g1_policy()->max_pause_time_ms(),

                                  cause);

       VMThread::execute(&op);

     } else {

       // Schedule a Full GC.

       VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);

       VMThread::execute(&op);

     }

   }

 }

 bool G1CollectedHeap::is_in(const void* p) const {

   if (_g1_committed.contains(p)) {

     HeapRegion* hr = _hrs->addr_to_region(p);

     return hr->is_in(p);

   } else {

     return _perm_gen->as_gen()->is_in(p);

   }

 }

 // Iteration functions.

 // Iterates an OopClosure over all ref-containing fields of objects

 // within a HeapRegion.

 class IterateOopClosureRegionClosure: public HeapRegionClosure {

   MemRegion _mr;

   OopClosure* _cl;

 public:

   IterateOopClosureRegionClosure(MemRegion mr, OopClosure* cl)

     : _mr(mr), _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     if (! r->continuesHumongous()) {

       r->oop_iterate(_cl);

     }

     return false;

   }

 };

 void G1CollectedHeap::oop_iterate(OopClosure* cl, bool do_perm) {

   IterateOopClosureRegionClosure blk(_g1_committed, cl);

   _hrs->iterate(&blk);

   if (do_perm) {

     perm_gen()->oop_iterate(cl);

   }

 }

 void G1CollectedHeap::oop_iterate(MemRegion mr, OopClosure* cl, bool do_perm) {

   IterateOopClosureRegionClosure blk(mr, cl);

   _hrs->iterate(&blk);

   if (do_perm) {

     perm_gen()->oop_iterate(cl);

   }

 }

 // Iterates an ObjectClosure over all objects within a HeapRegion.

 class IterateObjectClosureRegionClosure: public HeapRegionClosure {

   ObjectClosure* _cl;

 public:

   IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     if (! r->continuesHumongous()) {

       r->object_iterate(_cl);

     }

     return false;

   }

 };

 void G1CollectedHeap::object_iterate(ObjectClosure* cl, bool do_perm) {

   IterateObjectClosureRegionClosure blk(cl);

   _hrs->iterate(&blk);

   if (do_perm) {

     perm_gen()->object_iterate(cl);

   }

 }

 void G1CollectedHeap::object_iterate_since_last_GC(ObjectClosure* cl) {

   // FIXME: is this right?

   guarantee(false, "object_iterate_since_last_GC not supported by G1 heap");

 }

 // Calls a SpaceClosure on a HeapRegion.

 class SpaceClosureRegionClosure: public HeapRegionClosure {

   SpaceClosure* _cl;

 public:

   SpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     _cl->do_space(r);

     return false;

   }

 };

 void G1CollectedHeap::space_iterate(SpaceClosure* cl) {

   SpaceClosureRegionClosure blk(cl);

   _hrs->iterate(&blk);

 }

 void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) {

   _hrs->iterate(cl);

 }

 void G1CollectedHeap::heap_region_iterate_from(HeapRegion* r,

                                                HeapRegionClosure* cl) {

   _hrs->iterate_from(r, cl);

 }

 void

 G1CollectedHeap::heap_region_iterate_from(int idx, HeapRegionClosure* cl) {

   _hrs->iterate_from(idx, cl);

 }

 HeapRegion* G1CollectedHeap::region_at(size_t idx) { return _hrs->at(idx); }

 void

 G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl,

                                                  int worker,

                                                  jint claim_value) {

   const size_t regions = n_regions();

   const size_t worker_num = (G1CollectedHeap::use_parallel_gc_threads() ? ParallelGCThreads : 1);

   // try to spread out the starting points of the workers

   const size_t start_index = regions / worker_num * (size_t) worker;

   // each worker will actually look at all regions

   for (size_t count = 0; count < regions; ++count) {

     const size_t index = (start_index + count) % regions;

     assert(0 <= index && index < regions, "sanity");

     HeapRegion* r = region_at(index);

     // we'll ignore "continues humongous" regions (we'll process them

     // when we come across their corresponding "start humongous"

     // region) and regions already claimed

     if (r->claim_value() == claim_value || r->continuesHumongous()) {

       continue;

     }

     // OK, try to claim it

     if (r->claimHeapRegion(claim_value)) {

       // success!

       assert(!r->continuesHumongous(), "sanity");

       if (r->startsHumongous()) {

         // If the region is "starts humongous" we'll iterate over its

         // "continues humongous" first; in fact we'll do them

         // first. The order is important. In on case, calling the

         // closure on the "starts humongous" region might de-allocate

         // and clear all its "continues humongous" regions and, as a

         // result, we might end up processing them twice. So, we'll do

         // them first (notice: most closures will ignore them anyway) and

         // then we'll do the "starts humongous" region.

         for (size_t ch_index = index + 1; ch_index < regions; ++ch_index) {

           HeapRegion* chr = region_at(ch_index);

           // if the region has already been claimed or it's not

           // "continues humongous" we're done

           if (chr->claim_value() == claim_value ||

               !chr->continuesHumongous()) {

             break;

           }

           // Noone should have claimed it directly. We can given

           // that we claimed its "starts humongous" region.

           assert(chr->claim_value() != claim_value, "sanity");

           assert(chr->humongous_start_region() == r, "sanity");

           if (chr->claimHeapRegion(claim_value)) {

             // we should always be able to claim it; noone else should

             // be trying to claim this region

             bool res2 = cl->doHeapRegion(chr);

             assert(!res2, "Should not abort");

             // Right now, this holds (i.e., no closure that actually

             // does something with "continues humongous" regions

             // clears them). We might have to weaken it in the future,

             // but let's leave these two asserts here for extra safety.

             assert(chr->continuesHumongous(), "should still be the case");

             assert(chr->humongous_start_region() == r, "sanity");

           } else {

             guarantee(false, "we should not reach here");

           }

         }

       }

       assert(!r->continuesHumongous(), "sanity");

       bool res = cl->doHeapRegion(r);

       assert(!res, "Should not abort");

     }

   }

 }

 class ResetClaimValuesClosure: public HeapRegionClosure {

 public:

   bool doHeapRegion(HeapRegion* r) {

     r->set_claim_value(HeapRegion::InitialClaimValue);

     return false;

   }

 };

 void

 G1CollectedHeap::reset_heap_region_claim_values() {

   ResetClaimValuesClosure blk;

   heap_region_iterate(&blk);

 }

 #ifdef ASSERT

 // This checks whether all regions in the heap have the correct claim

 // value. I also piggy-backed on this a check to ensure that the

 // humongous_start_region() information on "continues humongous"

 // regions is correct.

 class CheckClaimValuesClosure : public HeapRegionClosure {

 private:

   jint _claim_value;

   size_t _failures;

   HeapRegion* _sh_region;

 public:

   CheckClaimValuesClosure(jint claim_value) :

     _claim_value(claim_value), _failures(0), _sh_region(NULL) { }

   bool doHeapRegion(HeapRegion* r) {

     if (r->claim_value() != _claim_value) {

       gclog_or_tty->print_cr("Region ["PTR_FORMAT","PTR_FORMAT"), "

                              "claim value = %d, should be %d",

                              r->bottom(), r->end(), r->claim_value(),

                              _claim_value);

       ++_failures;

     }

     if (!r->isHumongous()) {

       _sh_region = NULL;

     } else if (r->startsHumongous()) {

       _sh_region = r;

     } else if (r->continuesHumongous()) {

       if (r->humongous_start_region() != _sh_region) {

         gclog_or_tty->print_cr("Region ["PTR_FORMAT","PTR_FORMAT"), "

                                "HS = "PTR_FORMAT", should be "PTR_FORMAT,

                                r->bottom(), r->end(),

                                r->humongous_start_region(),

                                _sh_region);

         ++_failures;

       }

     }

     return false;

   }

   size_t failures() {

     return _failures;

   }

 };

 bool G1CollectedHeap::check_heap_region_claim_values(jint claim_value) {

   CheckClaimValuesClosure cl(claim_value);

   heap_region_iterate(&cl);

   return cl.failures() == 0;

 }

 #endif // ASSERT

 void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) {

   HeapRegion* r = g1_policy()->collection_set();

   while (r != NULL) {

     HeapRegion* next = r->next_in_collection_set();

     if (cl->doHeapRegion(r)) {

       cl->incomplete();

       return;

     }

     r = next;

   }

 }

 void G1CollectedHeap::collection_set_iterate_from(HeapRegion* r,

                                                   HeapRegionClosure *cl) {

   if (r == NULL) {

     // The CSet is empty so there's nothing to do.

     return;

   }

   assert(r->in_collection_set(),

          "Start region must be a member of the collection set.");

   HeapRegion* cur = r;

   while (cur != NULL) {

     HeapRegion* next = cur->next_in_collection_set();

     if (cl->doHeapRegion(cur) && false) {

       cl->incomplete();

       return;

     }

     cur = next;

   }

   cur = g1_policy()->collection_set();

   while (cur != r) {

     HeapRegion* next = cur->next_in_collection_set();

     if (cl->doHeapRegion(cur) && false) {

       cl->incomplete();

       return;

     }

     cur = next;

   }

 }

 CompactibleSpace* G1CollectedHeap::first_compactible_space() {

   return _hrs->length() > 0 ? _hrs->at(0) : NULL;

 }

 Space* G1CollectedHeap::space_containing(const void* addr) const {

   Space* res = heap_region_containing(addr);

   if (res == NULL)

     res = perm_gen()->space_containing(addr);

   return res;

 }

 HeapWord* G1CollectedHeap::block_start(const void* addr) const {

   Space* sp = space_containing(addr);

   if (sp != NULL) {

     return sp->block_start(addr);

   }

   return NULL;

 }

 size_t G1CollectedHeap::block_size(const HeapWord* addr) const {

   Space* sp = space_containing(addr);

   assert(sp != NULL, "block_size of address outside of heap");

   return sp->block_size(addr);

 }

 bool G1CollectedHeap::block_is_obj(const HeapWord* addr) const {

   Space* sp = space_containing(addr);

   return sp->block_is_obj(addr);

 }

 bool G1CollectedHeap::supports_tlab_allocation() const {

   return true;

 }

 size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const {

   return HeapRegion::GrainBytes;

 }

 size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const {

   // Return the remaining space in the cur alloc region, but not less than

   // the min TLAB size.

   // Also, this value can be at most the humongous object threshold,

   // since we can't allow tlabs to grow big enough to accomodate

   // humongous objects.

   // We need to store the cur alloc region locally, since it might change

   // between when we test for NULL and when we use it later.

   ContiguousSpace* cur_alloc_space = _cur_alloc_region;

   size_t max_tlab_size = _humongous_object_threshold_in_words * wordSize;

   if (cur_alloc_space == NULL) {

     return max_tlab_size;

   } else {

     return MIN2(MAX2(cur_alloc_space->free(), (size_t)MinTLABSize),

                 max_tlab_size);

   }

 }

 HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) {

   assert(!isHumongous(word_size),

          err_msg("a TLAB should not be of humongous size, "

                  "word_size = "SIZE_FORMAT, word_size));

   bool dummy;

   return G1CollectedHeap::mem_allocate(word_size, false, true, &dummy);

 }

 bool G1CollectedHeap::allocs_are_zero_filled() {

   return false;

 }

 size_t G1CollectedHeap::large_typearray_limit() {

   // FIXME

   return HeapRegion::GrainBytes/HeapWordSize;

 }

 size_t G1CollectedHeap::max_capacity() const {

   return g1_reserved_obj_bytes();

 }

 jlong G1CollectedHeap::millis_since_last_gc() {

   // assert(false, "NYI");

   return 0;

 }

 void G1CollectedHeap::prepare_for_verify() {

   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {

     ensure_parsability(false);

   }

   g1_rem_set()->prepare_for_verify();

 }

 class VerifyLivenessOopClosure: public OopClosure {

   G1CollectedHeap* g1h;

 public:

   VerifyLivenessOopClosure(G1CollectedHeap* _g1h) {

     g1h = _g1h;

   }

   void do_oop(narrowOop *p) { do_oop_work(p); }

   void do_oop(      oop *p) { do_oop_work(p); }

   template <class T> void do_oop_work(T *p) {

     oop obj = oopDesc::load_decode_heap_oop(p);

     guarantee(obj == NULL || !g1h->is_obj_dead(obj),

               "Dead object referenced by a not dead object");

   }

 };

 class VerifyObjsInRegionClosure: public ObjectClosure {

 private:

   G1CollectedHeap* _g1h;

   size_t _live_bytes;

   HeapRegion *_hr;

   bool _use_prev_marking;

 public:

   // use_prev_marking == true  -> use "prev" marking information,

   // use_prev_marking == false -> use "next" marking information

   VerifyObjsInRegionClosure(HeapRegion *hr, bool use_prev_marking)

     : _live_bytes(0), _hr(hr), _use_prev_marking(use_prev_marking) {

     _g1h = G1CollectedHeap::heap();

   }

   void do_object(oop o) {

     VerifyLivenessOopClosure isLive(_g1h);

     assert(o != NULL, "Huh?");

     if (!_g1h->is_obj_dead_cond(o, _use_prev_marking)) {

       o->oop_iterate(&isLive);

       if (!_hr->obj_allocated_since_prev_marking(o)) {

         size_t obj_size = o->size();    // Make sure we don't overflow

         _live_bytes += (obj_size * HeapWordSize);

       }

     }

   }

   size_t live_bytes() { return _live_bytes; }

 };

 class PrintObjsInRegionClosure : public ObjectClosure {

   HeapRegion *_hr;

   G1CollectedHeap *_g1;

 public:

   PrintObjsInRegionClosure(HeapRegion *hr) : _hr(hr) {

     _g1 = G1CollectedHeap::heap();

   };

   void do_object(oop o) {

     if (o != NULL) {

       HeapWord *start = (HeapWord *) o;

       size_t word_sz = o->size();

       gclog_or_tty->print("\nPrinting obj "PTR_FORMAT" of size " SIZE_FORMAT

                           " isMarkedPrev %d isMarkedNext %d isAllocSince %d\n",

                           (void*) o, word_sz,

                           _g1->isMarkedPrev(o),

                           _g1->isMarkedNext(o),

                           _hr->obj_allocated_since_prev_marking(o));

       HeapWord *end = start + word_sz;

       HeapWord *cur;

       int *val;

       for (cur = start; cur < end; cur++) {

         val = (int *) cur;

         gclog_or_tty->print("\t "PTR_FORMAT":"PTR_FORMAT"\n", val, *val);

       }

     }

   }

 };

 class VerifyRegionClosure: public HeapRegionClosure {

 private:

   bool _allow_dirty;

   bool _par;

   bool _use_prev_marking;

   bool _failures;

 public:

   // use_prev_marking == true  -> use "prev" marking information,

   // use_prev_marking == false -> use "next" marking information

   VerifyRegionClosure(bool allow_dirty, bool par, bool use_prev_marking)

     : _allow_dirty(allow_dirty),

       _par(par),

       _use_prev_marking(use_prev_marking),

       _failures(false) {}

   bool failures() {

     return _failures;

   }

   bool doHeapRegion(HeapRegion* r) {

     guarantee(_par || r->claim_value() == HeapRegion::InitialClaimValue,

               "Should be unclaimed at verify points.");

     if (!r->continuesHumongous()) {

       bool failures = false;

       r->verify(_allow_dirty, _use_prev_marking, &failures);

       if (failures) {

         _failures = true;

       } else {

         VerifyObjsInRegionClosure not_dead_yet_cl(r, _use_prev_marking);

         r->object_iterate(&not_dead_yet_cl);

         if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {

           gclog_or_tty->print_cr("["PTR_FORMAT","PTR_FORMAT"] "

                                  "max_live_bytes "SIZE_FORMAT" "

                                  "< calculated "SIZE_FORMAT,

                                  r->bottom(), r->end(),

                                  r->max_live_bytes(),

                                  not_dead_yet_cl.live_bytes());

           _failures = true;

         }

       }

     }

     return false; // stop the region iteration if we hit a failure

   }

 };

 class VerifyRootsClosure: public OopsInGenClosure {

 private:

   G1CollectedHeap* _g1h;

   bool             _use_prev_marking;

   bool             _failures;

 public:

   // use_prev_marking == true  -> use "prev" marking information,

   // use_prev_marking == false -> use "next" marking information

   VerifyRootsClosure(bool use_prev_marking) :

     _g1h(G1CollectedHeap::heap()),

     _use_prev_marking(use_prev_marking),

     _failures(false) { }

   bool failures() { return _failures; }

   template <class T> void do_oop_nv(T* p) {

     T heap_oop = oopDesc::load_heap_oop(p);

     if (!oopDesc::is_null(heap_oop)) {

       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);

       if (_g1h->is_obj_dead_cond(obj, _use_prev_marking)) {

         gclog_or_tty->print_cr("Root location "PTR_FORMAT" "

                               "points to dead obj "PTR_FORMAT, p, (void*) obj);

         obj->print_on(gclog_or_tty);

         _failures = true;

       }

     }

   }

   void do_oop(oop* p)       { do_oop_nv(p); }

   void do_oop(narrowOop* p) { do_oop_nv(p); }

 };

 // This is the task used for parallel heap verification.

 class G1ParVerifyTask: public AbstractGangTask {

 private:

   G1CollectedHeap* _g1h;

   bool _allow_dirty;

   bool _use_prev_marking;

   bool _failures;

 public:

   // use_prev_marking == true  -> use "prev" marking information,

   // use_prev_marking == false -> use "next" marking information

   G1ParVerifyTask(G1CollectedHeap* g1h, bool allow_dirty,

                   bool use_prev_marking) :

     AbstractGangTask("Parallel verify task"),

     _g1h(g1h),

     _allow_dirty(allow_dirty),

     _use_prev_marking(use_prev_marking),

     _failures(false) { }

   bool failures() {

     return _failures;

   }

   void work(int worker_i) {

     HandleMark hm;

     VerifyRegionClosure blk(_allow_dirty, true, _use_prev_marking);

     _g1h->heap_region_par_iterate_chunked(&blk, worker_i,

                                           HeapRegion::ParVerifyClaimValue);

     if (blk.failures()) {

       _failures = true;

     }

   }

 };

 void G1CollectedHeap::verify(bool allow_dirty, bool silent) {

   verify(allow_dirty, silent, /* use_prev_marking */ true);

 }

 void G1CollectedHeap::verify(bool allow_dirty,

                              bool silent,

                              bool use_prev_marking) {

   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {

     if (!silent) { gclog_or_tty->print("roots "); }

     VerifyRootsClosure rootsCl(use_prev_marking);

     CodeBlobToOopClosure blobsCl(&rootsCl, /*do_marking=*/ false);

     process_strong_roots(true,  // activate StrongRootsScope

                          false,

                          SharedHeap::SO_AllClasses,

                          &rootsCl,

                          &blobsCl,

                          &rootsCl);

     bool failures = rootsCl.failures();

     rem_set()->invalidate(perm_gen()->used_region(), false);

     if (!silent) { gclog_or_tty->print("heapRegions "); }

     if (GCParallelVerificationEnabled && ParallelGCThreads > 1) {

       assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue),

              "sanity check");

       G1ParVerifyTask task(this, allow_dirty, use_prev_marking);

       int n_workers = workers()->total_workers();

       set_par_threads(n_workers);

       workers()->run_task(&task);

       set_par_threads(0);

       if (task.failures()) {

         failures = true;

       }

       assert(check_heap_region_claim_values(HeapRegion::ParVerifyClaimValue),

              "sanity check");

       reset_heap_region_claim_values();

       assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue),

              "sanity check");

     } else {

       VerifyRegionClosure blk(allow_dirty, false, use_prev_marking);

       _hrs->iterate(&blk);

       if (blk.failures()) {

         failures = true;

       }

     }

     if (!silent) gclog_or_tty->print("remset ");

     rem_set()->verify();

     if (failures) {

       gclog_or_tty->print_cr("Heap:");

       print_on(gclog_or_tty, true /* extended */);

       gclog_or_tty->print_cr("");

 #ifndef PRODUCT

       if (VerifyDuringGC && G1VerifyDuringGCPrintReachable) {

         concurrent_mark()->print_reachable("at-verification-failure",

                                            use_prev_marking, false /* all */);

       }

 #endif

       gclog_or_tty->flush();

     }

     guarantee(!failures, "there should not have been any failures");

   } else {

     if (!silent) gclog_or_tty->print("(SKIPPING roots, heapRegions, remset) ");

   }

 }

 class PrintRegionClosure: public HeapRegionClosure {

   outputStream* _st;

 public:

   PrintRegionClosure(outputStream* st) : _st(st) {}

   bool doHeapRegion(HeapRegion* r) {

     r->print_on(_st);

     return false;

   }

 };

 void G1CollectedHeap::print() const { print_on(tty); }

 void G1CollectedHeap::print_on(outputStream* st) const {

   print_on(st, PrintHeapAtGCExtended);

 }

 void G1CollectedHeap::print_on(outputStream* st, bool extended) const {

   st->print(" %-20s", "garbage-first heap");

   st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K",

             capacity()/K, used_unlocked()/K);

   st->print(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")",

             _g1_storage.low_boundary(),

             _g1_storage.high(),

             _g1_storage.high_boundary());

   st->cr();

   st->print("  region size " SIZE_FORMAT "K, ",

             HeapRegion::GrainBytes/K);

   size_t young_regions = _young_list->length();

   st->print(SIZE_FORMAT " young (" SIZE_FORMAT "K), ",

             young_regions, young_regions * HeapRegion::GrainBytes / K);

   size_t survivor_regions = g1_policy()->recorded_survivor_regions();

   st->print(SIZE_FORMAT " survivors (" SIZE_FORMAT "K)",

             survivor_regions, survivor_regions * HeapRegion::GrainBytes / K);

   st->cr();

   perm()->as_gen()->print_on(st);

   if (extended) {

     st->cr();

     print_on_extended(st);

   }

 }

 void G1CollectedHeap::print_on_extended(outputStream* st) const {

   PrintRegionClosure blk(st);

   _hrs->iterate(&blk);

 }

 void G1CollectedHeap::print_gc_threads_on(outputStream* st) const {

   if (G1CollectedHeap::use_parallel_gc_threads()) {

     workers()->print_worker_threads_on(st);

   }

   _cmThread->print_on(st);

   st->cr();

   _cm->print_worker_threads_on(st);

   _cg1r->print_worker_threads_on(st);

   _czft->print_on(st);

   st->cr();

 }

 void G1CollectedHeap::gc_threads_do(ThreadClosure* tc) const {

   if (G1CollectedHeap::use_parallel_gc_threads()) {

     workers()->threads_do(tc);

   }

   tc->do_thread(_cmThread);

   _cg1r->threads_do(tc);

   tc->do_thread(_czft);

 }

 void G1CollectedHeap::print_tracing_info() const {

   // We'll overload this to mean "trace GC pause statistics."

   if (TraceGen0Time || TraceGen1Time) {

     // The "G1CollectorPolicy" is keeping track of these stats, so delegate

     // to that.

     g1_policy()->print_tracing_info();

   }

   if (G1SummarizeRSetStats) {

     g1_rem_set()->print_summary_info();

   }

   if (G1SummarizeConcMark) {

     concurrent_mark()->print_summary_info();

   }

   if (G1SummarizeZFStats) {

     ConcurrentZFThread::print_summary_info();

   }

   g1_policy()->print_yg_surv_rate_info();

   SpecializationStats::print();

 }

 int G1CollectedHeap::addr_to_arena_id(void* addr) const {

   HeapRegion* hr = heap_region_containing(addr);

   if (hr == NULL) {

     return 0;

   } else {

     return 1;

   }

 }

 G1CollectedHeap* G1CollectedHeap::heap() {

   assert(_sh->kind() == CollectedHeap::G1CollectedHeap,

          "not a garbage-first heap");

   return _g1h;

 }

 void G1CollectedHeap::gc_prologue(bool full /* Ignored */) {

   // always_do_update_barrier = false;

   assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");

   // Call allocation profiler

   AllocationProfiler::iterate_since_last_gc();

   // Fill TLAB's and such

   ensure_parsability(true);

 }

 void G1CollectedHeap::gc_epilogue(bool full /* Ignored */) {

   // FIXME: what is this about?

   // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled"

   // is set.

   COMPILER2_PRESENT(assert(DerivedPointerTable::is_empty(),

                         "derived pointer present"));

   // always_do_update_barrier = true;

 }

 void G1CollectedHeap::do_collection_pause() {

   assert(Heap_lock->owned_by_self(), "we assume we'reholding the Heap_lock");

   // Read the GC count while holding the Heap_lock

   // we need to do this _before_ wait_for_cleanup_complete(), to

   // ensure that we do not give up the heap lock and potentially

   // pick up the wrong count

   unsigned int gc_count_before = SharedHeap::heap()->total_collections();

   // Don't want to do a GC pause while cleanup is being completed!

   wait_for_cleanup_complete();

   g1_policy()->record_stop_world_start();

   {

     MutexUnlocker mu(Heap_lock);  // give up heap lock, execute gets it back

     VM_G1IncCollectionPause op(gc_count_before,

                                false, /* should_initiate_conc_mark */

                                g1_policy()->max_pause_time_ms(),

                                GCCause::_g1_inc_collection_pause);

     VMThread::execute(&op);

   }

 }

 void

 G1CollectedHeap::doConcurrentMark() {

   MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);

   if (!_cmThread->in_progress()) {