Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright 2001-2008 Sun Microsystems, Inc.  All Rights Reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,

  * CA 95054 USA or visit www.sun.com if you need additional information or

  * have any questions.

  *

  */

 #include "incls/_precompiled.incl"

 #include "incls/_g1CollectedHeap.cpp.incl"

 // turn it on so that the contents of the young list (scan-only /

 // to-be-collected) are printed at "strategic" points before / during

 // / after the collection --- this is useful for debugging

 #define SCAN_ONLY_VERBOSE 0

 // CURRENT STATUS

 // This file is under construction.  Search for "FIXME".

 // INVARIANTS/NOTES

 //

 // All allocation activity covered by the G1CollectedHeap interface is

 //   serialized by acquiring the HeapLock.  This happens in

 //   mem_allocate_work, which all such allocation functions call.

 //   (Note that this does not apply to TLAB allocation, which is not part

 //   of this interface: it is done by clients of this interface.)

 // Local to this file.

 // Finds the first HeapRegion.

 // No longer used, but might be handy someday.

 class FindFirstRegionClosure: public HeapRegionClosure {

   HeapRegion* _a_region;

 public:

   FindFirstRegionClosure() : _a_region(NULL) {}

   bool doHeapRegion(HeapRegion* r) {

     _a_region = r;

     return true;

   }

   HeapRegion* result() { return _a_region; }

 };

 class RefineCardTableEntryClosure: public CardTableEntryClosure {

   SuspendibleThreadSet* _sts;

   G1RemSet* _g1rs;

   ConcurrentG1Refine* _cg1r;

   bool _concurrent;

 public:

   RefineCardTableEntryClosure(SuspendibleThreadSet* sts,

                               G1RemSet* g1rs,

                               ConcurrentG1Refine* cg1r) :

     _sts(sts), _g1rs(g1rs), _cg1r(cg1r), _concurrent(true)

   {}

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     _g1rs->concurrentRefineOneCard(card_ptr, worker_i);

     if (_concurrent && _sts->should_yield()) {

       // Caller will actually yield.

       return false;

     }

     // Otherwise, we finished successfully; return true.

     return true;

   }

   void set_concurrent(bool b) { _concurrent = b; }

 };

 class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure {

   int _calls;

   G1CollectedHeap* _g1h;

   CardTableModRefBS* _ctbs;

   int _histo[256];

 public:

   ClearLoggedCardTableEntryClosure() :

     _calls(0)

   {

     _g1h = G1CollectedHeap::heap();

     _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

     for (int i = 0; i < 256; i++) _histo[i] = 0;

   }

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

       _calls++;

       unsigned char* ujb = (unsigned char*)card_ptr;

       int ind = (int)(*ujb);

       _histo[ind]++;

       *card_ptr = -1;

     }

     return true;

   }

   int calls() { return _calls; }

   void print_histo() {

     gclog_or_tty->print_cr("Card table value histogram:");

     for (int i = 0; i < 256; i++) {

       if (_histo[i] != 0) {

         gclog_or_tty->print_cr("  %d: %d", i, _histo[i]);

       }

     }

   }

 };

 class RedirtyLoggedCardTableEntryClosure: public CardTableEntryClosure {

   int _calls;

   G1CollectedHeap* _g1h;

   CardTableModRefBS* _ctbs;

 public:

   RedirtyLoggedCardTableEntryClosure() :

     _calls(0)

   {

     _g1h = G1CollectedHeap::heap();

     _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

   }

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

       _calls++;

       *card_ptr = 0;

     }

     return true;

   }

   int calls() { return _calls; }

 };

 class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure {

 public:

   bool do_card_ptr(jbyte* card_ptr, int worker_i) {

     *card_ptr = CardTableModRefBS::dirty_card_val();

     return true;

   }

 };

 YoungList::YoungList(G1CollectedHeap* g1h)

   : _g1h(g1h), _head(NULL),

     _scan_only_head(NULL), _scan_only_tail(NULL), _curr_scan_only(NULL),

     _length(0), _scan_only_length(0),

     _last_sampled_rs_lengths(0),

     _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0)

 {

   guarantee( check_list_empty(false), "just making sure..." );

 }

 void YoungList::push_region(HeapRegion *hr) {

   assert(!hr->is_young(), "should not already be young");

   assert(hr->get_next_young_region() == NULL, "cause it should!");

   hr->set_next_young_region(_head);

   _head = hr;

   hr->set_young();

   double yg_surv_rate = _g1h->g1_policy()->predict_yg_surv_rate((int)_length);

   ++_length;

 }

 void YoungList::add_survivor_region(HeapRegion* hr) {

   assert(hr->is_survivor(), "should be flagged as survivor region");

   assert(hr->get_next_young_region() == NULL, "cause it should!");

   hr->set_next_young_region(_survivor_head);

   if (_survivor_head == NULL) {

     _survivor_tail = hr;

   }

   _survivor_head = hr;

   ++_survivor_length;

 }

 HeapRegion* YoungList::pop_region() {

   while (_head != NULL) {

     assert( length() > 0, "list should not be empty" );

     HeapRegion* ret = _head;

     _head = ret->get_next_young_region();

     ret->set_next_young_region(NULL);

     --_length;

     assert(ret->is_young(), "region should be very young");

     // Replace 'Survivor' region type with 'Young'. So the region will

     // be treated as a young region and will not be 'confused' with

     // newly created survivor regions.

     if (ret->is_survivor()) {

       ret->set_young();

     }

     if (!ret->is_scan_only()) {

       return ret;

     }

     // scan-only, we'll add it to the scan-only list

     if (_scan_only_tail == NULL) {

       guarantee( _scan_only_head == NULL, "invariant" );

       _scan_only_head = ret;

       _curr_scan_only = ret;

     } else {

       guarantee( _scan_only_head != NULL, "invariant" );

       _scan_only_tail->set_next_young_region(ret);

     }

     guarantee( ret->get_next_young_region() == NULL, "invariant" );

     _scan_only_tail = ret;

     // no need to be tagged as scan-only any more

     ret->set_young();

     ++_scan_only_length;

   }

   assert( length() == 0, "list should be empty" );

   return NULL;

 }

 void YoungList::empty_list(HeapRegion* list) {

   while (list != NULL) {

     HeapRegion* next = list->get_next_young_region();

     list->set_next_young_region(NULL);

     list->uninstall_surv_rate_group();

     list->set_not_young();

     list = next;

   }

 }

 void YoungList::empty_list() {

   assert(check_list_well_formed(), "young list should be well formed");

   empty_list(_head);

   _head = NULL;

   _length = 0;

   empty_list(_scan_only_head);

   _scan_only_head = NULL;

   _scan_only_tail = NULL;

   _scan_only_length = 0;

   _curr_scan_only = NULL;

   empty_list(_survivor_head);

   _survivor_head = NULL;

   _survivor_tail = NULL;

   _survivor_length = 0;

   _last_sampled_rs_lengths = 0;

   assert(check_list_empty(false), "just making sure...");

 }

 bool YoungList::check_list_well_formed() {

   bool ret = true;

   size_t length = 0;

   HeapRegion* curr = _head;

   HeapRegion* last = NULL;

   while (curr != NULL) {

     if (!curr->is_young() || curr->is_scan_only()) {

       gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" "

                              "incorrectly tagged (%d, %d)",

                              curr->bottom(), curr->end(),

                              curr->is_young(), curr->is_scan_only());

       ret = false;

     }

     ++length;

     last = curr;

     curr = curr->get_next_young_region();

   }

   ret = ret && (length == _length);

   if (!ret) {

     gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!");

     gclog_or_tty->print_cr("###   list has %d entries, _length is %d",

                            length, _length);

   }

   bool scan_only_ret = true;

   length = 0;

   curr = _scan_only_head;

   last = NULL;

   while (curr != NULL) {

     if (!curr->is_young() || curr->is_scan_only()) {

       gclog_or_tty->print_cr("### SCAN-ONLY REGION "PTR_FORMAT"-"PTR_FORMAT" "

                              "incorrectly tagged (%d, %d)",

                              curr->bottom(), curr->end(),

                              curr->is_young(), curr->is_scan_only());

       scan_only_ret = false;

     }

     ++length;

     last = curr;

     curr = curr->get_next_young_region();

   }

   scan_only_ret = scan_only_ret && (length == _scan_only_length);

   if ( (last != _scan_only_tail) ||

        (_scan_only_head == NULL && _scan_only_tail != NULL) ||

        (_scan_only_head != NULL && _scan_only_tail == NULL) ) {

      gclog_or_tty->print_cr("## _scan_only_tail is set incorrectly");

      scan_only_ret = false;

   }

   if (_curr_scan_only != NULL && _curr_scan_only != _scan_only_head) {

     gclog_or_tty->print_cr("### _curr_scan_only is set incorrectly");

     scan_only_ret = false;

    }

   if (!scan_only_ret) {

     gclog_or_tty->print_cr("### SCAN-ONLY LIST seems not well formed!");

     gclog_or_tty->print_cr("###   list has %d entries, _scan_only_length is %d",

                   length, _scan_only_length);

   }

   return ret && scan_only_ret;

 }

 bool YoungList::check_list_empty(bool ignore_scan_only_list,

                                  bool check_sample) {

   bool ret = true;

   if (_length != 0) {

     gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %d",

                   _length);

     ret = false;

   }

   if (check_sample && _last_sampled_rs_lengths != 0) {

     gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths");

     ret = false;

   }

   if (_head != NULL) {

     gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head");

     ret = false;

   }

   if (!ret) {

     gclog_or_tty->print_cr("### YOUNG LIST does not seem empty");

   }

   if (ignore_scan_only_list)

     return ret;

   bool scan_only_ret = true;

   if (_scan_only_length != 0) {

     gclog_or_tty->print_cr("### SCAN-ONLY LIST should have 0 length, not %d",

                   _scan_only_length);

     scan_only_ret = false;

   }

   if (_scan_only_head != NULL) {

     gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL head");

      scan_only_ret = false;

   }

   if (_scan_only_tail != NULL) {

     gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL tail");

     scan_only_ret = false;

   }

   if (!scan_only_ret) {

     gclog_or_tty->print_cr("### SCAN-ONLY LIST does not seem empty");

   }

   return ret && scan_only_ret;

 }

 void

 YoungList::rs_length_sampling_init() {

   _sampled_rs_lengths = 0;

   _curr               = _head;

 }

 bool

 YoungList::rs_length_sampling_more() {

   return _curr != NULL;

 }

 void

 YoungList::rs_length_sampling_next() {

   assert( _curr != NULL, "invariant" );

   _sampled_rs_lengths += _curr->rem_set()->occupied();

   _curr = _curr->get_next_young_region();

   if (_curr == NULL) {

     _last_sampled_rs_lengths = _sampled_rs_lengths;

     // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths);

   }

 }

 void

 YoungList::reset_auxilary_lists() {

   // We could have just "moved" the scan-only list to the young list.

   // However, the scan-only list is ordered according to the region

   // age in descending order, so, by moving one entry at a time, we

   // ensure that it is recreated in ascending order.

   guarantee( is_empty(), "young list should be empty" );

   assert(check_list_well_formed(), "young list should be well formed");

   // Add survivor regions to SurvRateGroup.

   _g1h->g1_policy()->note_start_adding_survivor_regions();

   _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */);

   for (HeapRegion* curr = _survivor_head;

        curr != NULL;

        curr = curr->get_next_young_region()) {

     _g1h->g1_policy()->set_region_survivors(curr);

   }

   _g1h->g1_policy()->note_stop_adding_survivor_regions();

   if (_survivor_head != NULL) {

     _head           = _survivor_head;

     _length         = _survivor_length + _scan_only_length;

     _survivor_tail->set_next_young_region(_scan_only_head);

   } else {

     _head           = _scan_only_head;

     _length         = _scan_only_length;

   }

   for (HeapRegion* curr = _scan_only_head;

        curr != NULL;

        curr = curr->get_next_young_region()) {

     curr->recalculate_age_in_surv_rate_group();

   }

   _scan_only_head   = NULL;

   _scan_only_tail   = NULL;

   _scan_only_length = 0;

   _curr_scan_only   = NULL;

   _survivor_head    = NULL;

   _survivor_tail   = NULL;

   _survivor_length  = 0;

   _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */);

   assert(check_list_well_formed(), "young list should be well formed");

 }

 void YoungList::print() {

   HeapRegion* lists[] = {_head,   _scan_only_head, _survivor_head};

   const char* names[] = {"YOUNG", "SCAN-ONLY",     "SURVIVOR"};

   for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) {

     gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]);

     HeapRegion *curr = lists[list];

     if (curr == NULL)

       gclog_or_tty->print_cr("  empty");

     while (curr != NULL) {

       gclog_or_tty->print_cr("  [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, "

                              "age: %4d, y: %d, s-o: %d, surv: %d",

                              curr->bottom(), curr->end(),

                              curr->top(),

                              curr->prev_top_at_mark_start(),

                              curr->next_top_at_mark_start(),

                              curr->top_at_conc_mark_count(),

                              curr->age_in_surv_rate_group_cond(),

                              curr->is_young(),

                              curr->is_scan_only(),

                              curr->is_survivor());

       curr = curr->get_next_young_region();

     }

   }

   gclog_or_tty->print_cr("");

 }

 void G1CollectedHeap::stop_conc_gc_threads() {

   _cg1r->cg1rThread()->stop();

   _czft->stop();

   _cmThread->stop();

 }

 void G1CollectedHeap::check_ct_logs_at_safepoint() {

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set();

   // Count the dirty cards at the start.

   CountNonCleanMemRegionClosure count1(this);

   ct_bs->mod_card_iterate(&count1);

   int orig_count = count1.n();

   // First clear the logged cards.

   ClearLoggedCardTableEntryClosure clear;

   dcqs.set_closure(&clear);

   dcqs.apply_closure_to_all_completed_buffers();

   dcqs.iterate_closure_all_threads(false);

   clear.print_histo();

   // Now ensure that there's no dirty cards.

   CountNonCleanMemRegionClosure count2(this);

   ct_bs->mod_card_iterate(&count2);

   if (count2.n() != 0) {

     gclog_or_tty->print_cr("Card table has %d entries; %d originally",

                            count2.n(), orig_count);

   }

   guarantee(count2.n() == 0, "Card table should be clean.");

   RedirtyLoggedCardTableEntryClosure redirty;

   JavaThread::dirty_card_queue_set().set_closure(&redirty);

   dcqs.apply_closure_to_all_completed_buffers();

   dcqs.iterate_closure_all_threads(false);

   gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.",

                          clear.calls(), orig_count);

   guarantee(redirty.calls() == clear.calls(),

             "Or else mechanism is broken.");

   CountNonCleanMemRegionClosure count3(this);

   ct_bs->mod_card_iterate(&count3);

   if (count3.n() != orig_count) {

     gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.",

                            orig_count, count3.n());

     guarantee(count3.n() >= orig_count, "Should have restored them all.");

   }

   JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

 }

 // Private class members.

 G1CollectedHeap* G1CollectedHeap::_g1h;

 // Private methods.

 // Finds a HeapRegion that can be used to allocate a given size of block.

 HeapRegion* G1CollectedHeap::newAllocRegion_work(size_t word_size,

                                                  bool do_expand,

                                                  bool zero_filled) {

   ConcurrentZFThread::note_region_alloc();

   HeapRegion* res = alloc_free_region_from_lists(zero_filled);

   if (res == NULL && do_expand) {

     expand(word_size * HeapWordSize);

     res = alloc_free_region_from_lists(zero_filled);

     assert(res == NULL ||

            (!res->isHumongous() &&

             (!zero_filled ||

              res->zero_fill_state() == HeapRegion::Allocated)),

            "Alloc Regions must be zero filled (and non-H)");

   }

   if (res != NULL && res->is_empty()) _free_regions--;

   assert(res == NULL ||

          (!res->isHumongous() &&

           (!zero_filled ||

            res->zero_fill_state() == HeapRegion::Allocated)),

          "Non-young alloc Regions must be zero filled (and non-H)");

   if (G1TraceRegions) {

     if (res != NULL) {

       gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT", "PTR_FORMAT"], "

                              "top "PTR_FORMAT,

                              res->hrs_index(), res->bottom(), res->end(), res->top());

     }

   }

   return res;

 }

 HeapRegion* G1CollectedHeap::newAllocRegionWithExpansion(int purpose,

                                                          size_t word_size,

                                                          bool zero_filled) {

   HeapRegion* alloc_region = NULL;

   if (_gc_alloc_region_counts[purpose] < g1_policy()->max_regions(purpose)) {

     alloc_region = newAllocRegion_work(word_size, true, zero_filled);

     if (purpose == GCAllocForSurvived && alloc_region != NULL) {

       alloc_region->set_survivor();

     }

     ++_gc_alloc_region_counts[purpose];

   } else {

     g1_policy()->note_alloc_region_limit_reached(purpose);

   }

   return alloc_region;

 }

 // If could fit into free regions w/o expansion, try.

 // Otherwise, if can expand, do so.

 // Otherwise, if using ex regions might help, try with ex given back.

 HeapWord* G1CollectedHeap::humongousObjAllocate(size_t word_size) {

   assert(regions_accounted_for(), "Region leakage!");

   // We can't allocate H regions while cleanupComplete is running, since

   // some of the regions we find to be empty might not yet be added to the

   // unclean list.  (If we're already at a safepoint, this call is

   // unnecessary, not to mention wrong.)

   if (!SafepointSynchronize::is_at_safepoint())

     wait_for_cleanup_complete();

   size_t num_regions =

     round_to(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords;

   // Special case if < one region???

   // Remember the ft size.

   size_t x_size = expansion_regions();

   HeapWord* res = NULL;

   bool eliminated_allocated_from_lists = false;

   // Can the allocation potentially fit in the free regions?

   if (free_regions() >= num_regions) {

     res = _hrs->obj_allocate(word_size);

   }

   if (res == NULL) {

     // Try expansion.

     size_t fs = _hrs->free_suffix();

     if (fs + x_size >= num_regions) {

       expand((num_regions - fs) * HeapRegion::GrainBytes);

       res = _hrs->obj_allocate(word_size);

       assert(res != NULL, "This should have worked.");

     } else {

       // Expansion won't help.  Are there enough free regions if we get rid

       // of reservations?

       size_t avail = free_regions();

       if (avail >= num_regions) {

         res = _hrs->obj_allocate(word_size);

         if (res != NULL) {

           remove_allocated_regions_from_lists();

           eliminated_allocated_from_lists = true;

         }

       }

     }

   }

   if (res != NULL) {

     // Increment by the number of regions allocated.

     // FIXME: Assumes regions all of size GrainBytes.

 #ifndef PRODUCT

     mr_bs()->verify_clean_region(MemRegion(res, res + num_regions *

                                            HeapRegion::GrainWords));

 #endif

     if (!eliminated_allocated_from_lists)

       remove_allocated_regions_from_lists();

     _summary_bytes_used += word_size * HeapWordSize;

     _free_regions -= num_regions;

     _num_humongous_regions += (int) num_regions;

   }

   assert(regions_accounted_for(), "Region Leakage");

   return res;

 }

 HeapWord*

 G1CollectedHeap::attempt_allocation_slow(size_t word_size,

                                          bool permit_collection_pause) {

   HeapWord* res = NULL;

   HeapRegion* allocated_young_region = NULL;

   assert( SafepointSynchronize::is_at_safepoint() ||

           Heap_lock->owned_by_self(), "pre condition of the call" );

   if (isHumongous(word_size)) {

     // Allocation of a humongous object can, in a sense, complete a

     // partial region, if the previous alloc was also humongous, and

     // caused the test below to succeed.

     if (permit_collection_pause)

       do_collection_pause_if_appropriate(word_size);

     res = humongousObjAllocate(word_size);

     assert(_cur_alloc_region == NULL

            || !_cur_alloc_region->isHumongous(),

            "Prevent a regression of this bug.");

   } else {

     // We may have concurrent cleanup working at the time. Wait for it

     // to complete. In the future we would probably want to make the

     // concurrent cleanup truly concurrent by decoupling it from the

     // allocation.

     if (!SafepointSynchronize::is_at_safepoint())

       wait_for_cleanup_complete();

     // If we do a collection pause, this will be reset to a non-NULL

     // value.  If we don't, nulling here ensures that we allocate a new

     // region below.

     if (_cur_alloc_region != NULL) {

       // We're finished with the _cur_alloc_region.

       _summary_bytes_used += _cur_alloc_region->used();

       _cur_alloc_region = NULL;

     }

     assert(_cur_alloc_region == NULL, "Invariant.");

     // Completion of a heap region is perhaps a good point at which to do

     // a collection pause.

     if (permit_collection_pause)

       do_collection_pause_if_appropriate(word_size);

     // Make sure we have an allocation region available.

     if (_cur_alloc_region == NULL) {

       if (!SafepointSynchronize::is_at_safepoint())

         wait_for_cleanup_complete();

       bool next_is_young = should_set_young_locked();

       // If the next region is not young, make sure it's zero-filled.

       _cur_alloc_region = newAllocRegion(word_size, !next_is_young);

       if (_cur_alloc_region != NULL) {

         _summary_bytes_used -= _cur_alloc_region->used();

         if (next_is_young) {

           set_region_short_lived_locked(_cur_alloc_region);

           allocated_young_region = _cur_alloc_region;

         }

       }

     }

     assert(_cur_alloc_region == NULL || !_cur_alloc_region->isHumongous(),

            "Prevent a regression of this bug.");

     // Now retry the allocation.

     if (_cur_alloc_region != NULL) {

       res = _cur_alloc_region->allocate(word_size);

     }

   }

   // NOTE: fails frequently in PRT

   assert(regions_accounted_for(), "Region leakage!");

   if (res != NULL) {

     if (!SafepointSynchronize::is_at_safepoint()) {

       assert( permit_collection_pause, "invariant" );

       assert( Heap_lock->owned_by_self(), "invariant" );

       Heap_lock->unlock();

     }

     if (allocated_young_region != NULL) {

       HeapRegion* hr = allocated_young_region;

       HeapWord* bottom = hr->bottom();

       HeapWord* end = hr->end();

       MemRegion mr(bottom, end);

       ((CardTableModRefBS*)_g1h->barrier_set())->dirty(mr);

     }

   }

   assert( SafepointSynchronize::is_at_safepoint() ||

           (res == NULL && Heap_lock->owned_by_self()) ||

           (res != NULL && !Heap_lock->owned_by_self()),

           "post condition of the call" );

   return res;

 }

 HeapWord*

 G1CollectedHeap::mem_allocate(size_t word_size,

                               bool   is_noref,

                               bool   is_tlab,

                               bool* gc_overhead_limit_was_exceeded) {

   debug_only(check_for_valid_allocation_state());

   assert(no_gc_in_progress(), "Allocation during gc not allowed");

   HeapWord* result = NULL;

   // Loop until the allocation is satisified,

   // or unsatisfied after GC.

   for (int try_count = 1; /* return or throw */; try_count += 1) {

     int gc_count_before;

     {

       Heap_lock->lock();

       result = attempt_allocation(word_size);

       if (result != NULL) {

         // attempt_allocation should have unlocked the heap lock

         assert(is_in(result), "result not in heap");

         return result;

       }

       // Read the gc count while the heap lock is held.

       gc_count_before = SharedHeap::heap()->total_collections();

       Heap_lock->unlock();

     }

     // Create the garbage collection operation...

     VM_G1CollectForAllocation op(word_size,

                                  gc_count_before);

     // ...and get the VM thread to execute it.

     VMThread::execute(&op);

     if (op.prologue_succeeded()) {

       result = op.result();

       assert(result == NULL || is_in(result), "result not in heap");

       return result;

     }

     // Give a warning if we seem to be looping forever.

     if ((QueuedAllocationWarningCount > 0) &&

         (try_count % QueuedAllocationWarningCount == 0)) {

       warning("G1CollectedHeap::mem_allocate_work retries %d times",

               try_count);

     }

   }

 }

 void G1CollectedHeap::abandon_cur_alloc_region() {

   if (_cur_alloc_region != NULL) {

     // We're finished with the _cur_alloc_region.

     if (_cur_alloc_region->is_empty()) {

       _free_regions++;

       free_region(_cur_alloc_region);

     } else {

       _summary_bytes_used += _cur_alloc_region->used();

     }

     _cur_alloc_region = NULL;

   }

 }

 class PostMCRemSetClearClosure: public HeapRegionClosure {

   ModRefBarrierSet* _mr_bs;

 public:

   PostMCRemSetClearClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}

   bool doHeapRegion(HeapRegion* r) {

     r->reset_gc_time_stamp();

     if (r->continuesHumongous())

       return false;

     HeapRegionRemSet* hrrs = r->rem_set();

     if (hrrs != NULL) hrrs->clear();

     // You might think here that we could clear just the cards

     // corresponding to the used region.  But no: if we leave a dirty card

     // in a region we might allocate into, then it would prevent that card

     // from being enqueued, and cause it to be missed.

     // Re: the performance cost: we shouldn't be doing full GC anyway!

     _mr_bs->clear(MemRegion(r->bottom(), r->end()));

     return false;

   }

 };

 class PostMCRemSetInvalidateClosure: public HeapRegionClosure {

   ModRefBarrierSet* _mr_bs;

 public:

   PostMCRemSetInvalidateClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}

   bool doHeapRegion(HeapRegion* r) {

     if (r->continuesHumongous()) return false;

     if (r->used_region().word_size() != 0) {

       _mr_bs->invalidate(r->used_region(), true /*whole heap*/);

     }

     return false;

   }

 };

 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {

   G1CollectedHeap*   _g1h;

   UpdateRSOopClosure _cl;

   int                _worker_i;

 public:

   RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, int worker_i = 0) :

     _cl(g1->g1_rem_set()->as_HRInto_G1RemSet(), worker_i),

     _worker_i(worker_i),

     _g1h(g1)

   { }

   bool doHeapRegion(HeapRegion* r) {

     if (!r->continuesHumongous()) {

       _cl.set_from(r);

       r->oop_iterate(&_cl);

     }

     return false;

   }

 };

 class ParRebuildRSTask: public AbstractGangTask {

   G1CollectedHeap* _g1;

 public:

   ParRebuildRSTask(G1CollectedHeap* g1)

     : AbstractGangTask("ParRebuildRSTask"),

       _g1(g1)

   { }

   void work(int i) {

     RebuildRSOutOfRegionClosure rebuild_rs(_g1, i);

     _g1->heap_region_par_iterate_chunked(&rebuild_rs, i,

                                          HeapRegion::RebuildRSClaimValue);

   }

 };

 void G1CollectedHeap::do_collection(bool full, bool clear_all_soft_refs,

                                     size_t word_size) {

   ResourceMark rm;

   if (full && DisableExplicitGC) {

     gclog_or_tty->print("\n\n\nDisabling Explicit GC\n\n\n");

     return;

   }

   assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");

   assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread");

   if (GC_locker::is_active()) {

     return; // GC is disabled (e.g. JNI GetXXXCritical operation)

   }

   {

     IsGCActiveMark x;

     // Timing

     gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);

     TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);

     TraceTime t(full ? "Full GC (System.gc())" : "Full GC", PrintGC, true, gclog_or_tty);

     double start = os::elapsedTime();

     GCOverheadReporter::recordSTWStart(start);

     g1_policy()->record_full_collection_start();

     gc_prologue(true);

     increment_total_collections();

     size_t g1h_prev_used = used();

     assert(used() == recalculate_used(), "Should be equal");

     if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       prepare_for_verify();

       gclog_or_tty->print(" VerifyBeforeGC:");

       Universe::verify(true);

     }

     assert(regions_accounted_for(), "Region leakage!");

     COMPILER2_PRESENT(DerivedPointerTable::clear());

     // We want to discover references, but not process them yet.

     // This mode is disabled in

     // instanceRefKlass::process_discovered_references if the

     // generation does some collection work, or

     // instanceRefKlass::enqueue_discovered_references if the

     // generation returns without doing any work.

     ref_processor()->disable_discovery();

     ref_processor()->abandon_partial_discovery();

     ref_processor()->verify_no_references_recorded();

     // Abandon current iterations of concurrent marking and concurrent

     // refinement, if any are in progress.

     concurrent_mark()->abort();

     // Make sure we'll choose a new allocation region afterwards.

     abandon_cur_alloc_region();

     assert(_cur_alloc_region == NULL, "Invariant.");

     g1_rem_set()->as_HRInto_G1RemSet()->cleanupHRRS();

     tear_down_region_lists();

     set_used_regions_to_need_zero_fill();

     if (g1_policy()->in_young_gc_mode()) {

       empty_young_list();

       g1_policy()->set_full_young_gcs(true);

     }

     // Temporarily make reference _discovery_ single threaded (non-MT).

     ReferenceProcessorMTMutator rp_disc_ser(ref_processor(), false);

     // Temporarily make refs discovery atomic

     ReferenceProcessorAtomicMutator rp_disc_atomic(ref_processor(), true);

     // Temporarily clear _is_alive_non_header

     ReferenceProcessorIsAliveMutator rp_is_alive_null(ref_processor(), NULL);

     ref_processor()->enable_discovery();

     ref_processor()->setup_policy(clear_all_soft_refs);

     // Do collection work

     {

       HandleMark hm;  // Discard invalid handles created during gc

       G1MarkSweep::invoke_at_safepoint(ref_processor(), clear_all_soft_refs);

     }

     // Because freeing humongous regions may have added some unclean

     // regions, it is necessary to tear down again before rebuilding.

     tear_down_region_lists();

     rebuild_region_lists();

     _summary_bytes_used = recalculate_used();

     ref_processor()->enqueue_discovered_references();

     COMPILER2_PRESENT(DerivedPointerTable::update_pointers());

     if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       gclog_or_tty->print(" VerifyAfterGC:");

       Universe::verify(false);

     }

     NOT_PRODUCT(ref_processor()->verify_no_references_recorded());

     reset_gc_time_stamp();

     // Since everything potentially moved, we will clear all remembered

     // sets, and clear all cards.  Later we will rebuild remebered

     // sets. We will also reset the GC time stamps of the regions.

     PostMCRemSetClearClosure rs_clear(mr_bs());

     heap_region_iterate(&rs_clear);

     // Resize the heap if necessary.

     resize_if_necessary_after_full_collection(full ? 0 : word_size);

     if (_cg1r->use_cache()) {

       _cg1r->clear_and_record_card_counts();

       _cg1r->clear_hot_cache();

     }

     // Rebuild remembered sets of all regions.

     if (ParallelGCThreads > 0) {

       ParRebuildRSTask rebuild_rs_task(this);

       assert(check_heap_region_claim_values(

              HeapRegion::InitialClaimValue), "sanity check");

       set_par_threads(workers()->total_workers());

       workers()->run_task(&rebuild_rs_task);

       set_par_threads(0);

       assert(check_heap_region_claim_values(

              HeapRegion::RebuildRSClaimValue), "sanity check");

       reset_heap_region_claim_values();

     } else {

       RebuildRSOutOfRegionClosure rebuild_rs(this);

       heap_region_iterate(&rebuild_rs);

     }

     if (PrintGC) {

       print_size_transition(gclog_or_tty, g1h_prev_used, used(), capacity());

     }

     if (true) { // FIXME

       // Ask the permanent generation to adjust size for full collections

       perm()->compute_new_size();

     }

     double end = os::elapsedTime();

     GCOverheadReporter::recordSTWEnd(end);

     g1_policy()->record_full_collection_end();

 #ifdef TRACESPINNING

     ParallelTaskTerminator::print_termination_counts();

 #endif

     gc_epilogue(true);

     // Abandon concurrent refinement.  This must happen last: in the

     // dirty-card logging system, some cards may be dirty by weak-ref

     // processing, and may be enqueued.  But the whole card table is

     // dirtied, so this should abandon those logs, and set "do_traversal"

     // to true.

     concurrent_g1_refine()->set_pya_restart();

     assert(!G1DeferredRSUpdate

            || (G1DeferredRSUpdate && (dirty_card_queue_set().completed_buffers_num() == 0)), "Should not be any");

     assert(regions_accounted_for(), "Region leakage!");

   }

   if (g1_policy()->in_young_gc_mode()) {

     _young_list->reset_sampled_info();

     assert( check_young_list_empty(false, false),

             "young list should be empty at this point");

   }

 }

 void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {

   do_collection(true, clear_all_soft_refs, 0);

 }

 // This code is mostly copied from TenuredGeneration.

 void

 G1CollectedHeap::

 resize_if_necessary_after_full_collection(size_t word_size) {

   assert(MinHeapFreeRatio <= MaxHeapFreeRatio, "sanity check");

   // Include the current allocation, if any, and bytes that will be

   // pre-allocated to support collections, as "used".

   const size_t used_after_gc = used();

   const size_t capacity_after_gc = capacity();

   const size_t free_after_gc = capacity_after_gc - used_after_gc;

   // We don't have floating point command-line arguments

   const double minimum_free_percentage = (double) MinHeapFreeRatio / 100;

   const double maximum_used_percentage = 1.0 - minimum_free_percentage;

   const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;

   const double minimum_used_percentage = 1.0 - maximum_free_percentage;

   size_t minimum_desired_capacity = (size_t) (used_after_gc / maximum_used_percentage);

   size_t maximum_desired_capacity = (size_t) (used_after_gc / minimum_used_percentage);

   // Don't shrink less than the initial size.

   minimum_desired_capacity =

     MAX2(minimum_desired_capacity,

          collector_policy()->initial_heap_byte_size());

   maximum_desired_capacity =

     MAX2(maximum_desired_capacity,

          collector_policy()->initial_heap_byte_size());

   // We are failing here because minimum_desired_capacity is

   assert(used_after_gc <= minimum_desired_capacity, "sanity check");

   assert(minimum_desired_capacity <= maximum_desired_capacity, "sanity check");

   if (PrintGC && Verbose) {

     const double free_percentage = ((double)free_after_gc) / capacity();

     gclog_or_tty->print_cr("Computing new size after full GC ");

     gclog_or_tty->print_cr("  "

                            "  minimum_free_percentage: %6.2f",

                            minimum_free_percentage);

     gclog_or_tty->print_cr("  "

                            "  maximum_free_percentage: %6.2f",

                            maximum_free_percentage);

     gclog_or_tty->print_cr("  "

                            "  capacity: %6.1fK"

                            "  minimum_desired_capacity: %6.1fK"

                            "  maximum_desired_capacity: %6.1fK",

                            capacity() / (double) K,

                            minimum_desired_capacity / (double) K,

                            maximum_desired_capacity / (double) K);

     gclog_or_tty->print_cr("  "

                            "   free_after_gc   : %6.1fK"

                            "   used_after_gc   : %6.1fK",

                            free_after_gc / (double) K,

                            used_after_gc / (double) K);

     gclog_or_tty->print_cr("  "

                            "   free_percentage: %6.2f",

                            free_percentage);

   }

   if (capacity() < minimum_desired_capacity) {

     // Don't expand unless it's significant

     size_t expand_bytes = minimum_desired_capacity - capacity_after_gc;

     expand(expand_bytes);

     if (PrintGC && Verbose) {

       gclog_or_tty->print_cr("    expanding:"

                              "  minimum_desired_capacity: %6.1fK"

                              "  expand_bytes: %6.1fK",

                              minimum_desired_capacity / (double) K,

                              expand_bytes / (double) K);

     }

     // No expansion, now see if we want to shrink

   } else if (capacity() > maximum_desired_capacity) {

     // Capacity too large, compute shrinking size

     size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity;

     shrink(shrink_bytes);

     if (PrintGC && Verbose) {

       gclog_or_tty->print_cr("  "

                              "  shrinking:"

                              "  initSize: %.1fK"

                              "  maximum_desired_capacity: %.1fK",

                              collector_policy()->initial_heap_byte_size() / (double) K,

                              maximum_desired_capacity / (double) K);

       gclog_or_tty->print_cr("  "

                              "  shrink_bytes: %.1fK",

                              shrink_bytes / (double) K);

     }

   }

 }

 HeapWord*

 G1CollectedHeap::satisfy_failed_allocation(size_t word_size) {

   HeapWord* result = NULL;

   // In a G1 heap, we're supposed to keep allocation from failing by

   // incremental pauses.  Therefore, at least for now, we'll favor

   // expansion over collection.  (This might change in the future if we can

   // do something smarter than full collection to satisfy a failed alloc.)

   result = expand_and_allocate(word_size);

   if (result != NULL) {

     assert(is_in(result), "result not in heap");

     return result;

   }

   // OK, I guess we have to try collection.

   do_collection(false, false, word_size);

   result = attempt_allocation(word_size, /*permit_collection_pause*/false);

   if (result != NULL) {

     assert(is_in(result), "result not in heap");

     return result;

   }

   // Try collecting soft references.

   do_collection(false, true, word_size);

   result = attempt_allocation(word_size, /*permit_collection_pause*/false);

   if (result != NULL) {

     assert(is_in(result), "result not in heap");

     return result;

   }

   // What else?  We might try synchronous finalization later.  If the total

   // space available is large enough for the allocation, then a more

   // complete compaction phase than we've tried so far might be

   // appropriate.

   return NULL;

 }

 // Attempting to expand the heap sufficiently

 // to support an allocation of the given "word_size".  If

 // successful, perform the allocation and return the address of the

 // allocated block, or else "NULL".

 HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size) {

   size_t expand_bytes = word_size * HeapWordSize;

   if (expand_bytes < MinHeapDeltaBytes) {

     expand_bytes = MinHeapDeltaBytes;

   }

   expand(expand_bytes);

   assert(regions_accounted_for(), "Region leakage!");

   HeapWord* result = attempt_allocation(word_size, false /* permit_collection_pause */);

   return result;

 }

 size_t G1CollectedHeap::free_region_if_totally_empty(HeapRegion* hr) {

   size_t pre_used = 0;

   size_t cleared_h_regions = 0;

   size_t freed_regions = 0;

   UncleanRegionList local_list;

   free_region_if_totally_empty_work(hr, pre_used, cleared_h_regions,

                                     freed_regions, &local_list);

   finish_free_region_work(pre_used, cleared_h_regions, freed_regions,

                           &local_list);

   return pre_used;

 }

 void

 G1CollectedHeap::free_region_if_totally_empty_work(HeapRegion* hr,

                                                    size_t& pre_used,

                                                    size_t& cleared_h,

                                                    size_t& freed_regions,

                                                    UncleanRegionList* list,

                                                    bool par) {

   assert(!hr->continuesHumongous(), "should have filtered these out");

   size_t res = 0;

   if (!hr->popular() && hr->used() > 0 && hr->garbage_bytes() == hr->used()) {

     if (!hr->is_young()) {

       if (G1PolicyVerbose > 0)

         gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)"

                                " during cleanup", hr, hr->used());

       free_region_work(hr, pre_used, cleared_h, freed_regions, list, par);

     }

   }

 }

 // FIXME: both this and shrink could probably be more efficient by

 // doing one "VirtualSpace::expand_by" call rather than several.

 void G1CollectedHeap::expand(size_t expand_bytes) {

   size_t old_mem_size = _g1_storage.committed_size();

   // We expand by a minimum of 1K.

   expand_bytes = MAX2(expand_bytes, (size_t)K);

   size_t aligned_expand_bytes =

     ReservedSpace::page_align_size_up(expand_bytes);

   aligned_expand_bytes = align_size_up(aligned_expand_bytes,

                                        HeapRegion::GrainBytes);

   expand_bytes = aligned_expand_bytes;

   while (expand_bytes > 0) {

     HeapWord* base = (HeapWord*)_g1_storage.high();

     // Commit more storage.

     bool successful = _g1_storage.expand_by(HeapRegion::GrainBytes);

     if (!successful) {

         expand_bytes = 0;

     } else {

       expand_bytes -= HeapRegion::GrainBytes;

       // Expand the committed region.

       HeapWord* high = (HeapWord*) _g1_storage.high();

       _g1_committed.set_end(high);

       // Create a new HeapRegion.

       MemRegion mr(base, high);

       bool is_zeroed = !_g1_max_committed.contains(base);

       HeapRegion* hr = new HeapRegion(_bot_shared, mr, is_zeroed);

       // Now update max_committed if necessary.

       _g1_max_committed.set_end(MAX2(_g1_max_committed.end(), high));

       // Add it to the HeapRegionSeq.

       _hrs->insert(hr);

       // Set the zero-fill state, according to whether it's already

       // zeroed.

       {

         MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);

         if (is_zeroed) {

           hr->set_zero_fill_complete();

           put_free_region_on_list_locked(hr);

         } else {

           hr->set_zero_fill_needed();

           put_region_on_unclean_list_locked(hr);

         }

       }

       _free_regions++;

       // And we used up an expansion region to create it.

       _expansion_regions--;

       // Tell the cardtable about it.

       Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);

       // And the offset table as well.

       _bot_shared->resize(_g1_committed.word_size());

     }

   }

   if (Verbose && PrintGC) {

     size_t new_mem_size = _g1_storage.committed_size();

     gclog_or_tty->print_cr("Expanding garbage-first heap from %ldK by %ldK to %ldK",

                            old_mem_size/K, aligned_expand_bytes/K,

                            new_mem_size/K);

   }

 }

 void G1CollectedHeap::shrink_helper(size_t shrink_bytes)

 {

   size_t old_mem_size = _g1_storage.committed_size();

   size_t aligned_shrink_bytes =

     ReservedSpace::page_align_size_down(shrink_bytes);

   aligned_shrink_bytes = align_size_down(aligned_shrink_bytes,

                                          HeapRegion::GrainBytes);

   size_t num_regions_deleted = 0;

   MemRegion mr = _hrs->shrink_by(aligned_shrink_bytes, num_regions_deleted);

   assert(mr.end() == (HeapWord*)_g1_storage.high(), "Bad shrink!");

   if (mr.byte_size() > 0)

     _g1_storage.shrink_by(mr.byte_size());

   assert(mr.start() == (HeapWord*)_g1_storage.high(), "Bad shrink!");

   _g1_committed.set_end(mr.start());

   _free_regions -= num_regions_deleted;

   _expansion_regions += num_regions_deleted;

   // Tell the cardtable about it.

   Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);

   // And the offset table as well.

   _bot_shared->resize(_g1_committed.word_size());

   HeapRegionRemSet::shrink_heap(n_regions());

   if (Verbose && PrintGC) {

     size_t new_mem_size = _g1_storage.committed_size();

     gclog_or_tty->print_cr("Shrinking garbage-first heap from %ldK by %ldK to %ldK",

                            old_mem_size/K, aligned_shrink_bytes/K,

                            new_mem_size/K);

   }

 }

 void G1CollectedHeap::shrink(size_t shrink_bytes) {

   release_gc_alloc_regions();

   tear_down_region_lists();  // We will rebuild them in a moment.

   shrink_helper(shrink_bytes);

   rebuild_region_lists();

 }

 // Public methods.

 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away

 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list

 #endif // _MSC_VER

 G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) :

   SharedHeap(policy_),

   _g1_policy(policy_),

   _ref_processor(NULL),

   _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)),

   _bot_shared(NULL),

   _par_alloc_during_gc_lock(Mutex::leaf, "par alloc during GC lock"),

   _objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL),

   _evac_failure_scan_stack(NULL) ,

   _mark_in_progress(false),

   _cg1r(NULL), _czft(NULL), _summary_bytes_used(0),

   _cur_alloc_region(NULL),

   _refine_cte_cl(NULL),

   _free_region_list(NULL), _free_region_list_size(0),

   _free_regions(0),

   _popular_object_boundary(NULL),

   _cur_pop_hr_index(0),

   _popular_regions_to_be_evacuated(NULL),

   _pop_obj_rc_at_copy(),

   _full_collection(false),

   _unclean_region_list(),

   _unclean_regions_coming(false),

   _young_list(new YoungList(this)),

   _gc_time_stamp(0),

   _surviving_young_words(NULL),

   _in_cset_fast_test(NULL),

   _in_cset_fast_test_base(NULL)

 {

   _g1h = this; // To catch bugs.

   if (_process_strong_tasks == NULL || !_process_strong_tasks->valid()) {

     vm_exit_during_initialization("Failed necessary allocation.");

   }

   int n_queues = MAX2((int)ParallelGCThreads, 1);

   _task_queues = new RefToScanQueueSet(n_queues);

   int n_rem_sets = HeapRegionRemSet::num_par_rem_sets();

   assert(n_rem_sets > 0, "Invariant.");

   HeapRegionRemSetIterator** iter_arr =

     NEW_C_HEAP_ARRAY(HeapRegionRemSetIterator*, n_queues);

   for (int i = 0; i < n_queues; i++) {

     iter_arr[i] = new HeapRegionRemSetIterator();

   }

   _rem_set_iterator = iter_arr;

   for (int i = 0; i < n_queues; i++) {

     RefToScanQueue* q = new RefToScanQueue();

     q->initialize();

     _task_queues->register_queue(i, q);

   }

   for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {

     _gc_alloc_regions[ap]       = NULL;

     _gc_alloc_region_counts[ap] = 0;

   }

   guarantee(_task_queues != NULL, "task_queues allocation failure.");

 }

 jint G1CollectedHeap::initialize() {

   os::enable_vtime();

   // Necessary to satisfy locking discipline assertions.

   MutexLocker x(Heap_lock);

   // While there are no constraints in the GC code that HeapWordSize

   // be any particular value, there are multiple other areas in the

   // system which believe this to be true (e.g. oop->object_size in some

   // cases incorrectly returns the size in wordSize units rather than

   // HeapWordSize).

   guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize");

   size_t init_byte_size = collector_policy()->initial_heap_byte_size();

   size_t max_byte_size = collector_policy()->max_heap_byte_size();

   // Ensure that the sizes are properly aligned.

   Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap");

   Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap");

   // We allocate this in any case, but only do no work if the command line

   // param is off.

   _cg1r = new ConcurrentG1Refine();

   // Reserve the maximum.

   PermanentGenerationSpec* pgs = collector_policy()->permanent_generation();

   // Includes the perm-gen.

   ReservedSpace heap_rs(max_byte_size + pgs->max_size(),

                         HeapRegion::GrainBytes,

                         false /*ism*/);

   if (!heap_rs.is_reserved()) {

     vm_exit_during_initialization("Could not reserve enough space for object heap");

     return JNI_ENOMEM;

   }

   // It is important to do this in a way such that concurrent readers can't

   // temporarily think somethings in the heap.  (I've actually seen this

   // happen in asserts: DLD.)

   _reserved.set_word_size(0);

   _reserved.set_start((HeapWord*)heap_rs.base());

   _reserved.set_end((HeapWord*)(heap_rs.base() + heap_rs.size()));

   _expansion_regions = max_byte_size/HeapRegion::GrainBytes;

   _num_humongous_regions = 0;

   // Create the gen rem set (and barrier set) for the entire reserved region.

   _rem_set = collector_policy()->create_rem_set(_reserved, 2);

   set_barrier_set(rem_set()->bs());

   if (barrier_set()->is_a(BarrierSet::ModRef)) {

     _mr_bs = (ModRefBarrierSet*)_barrier_set;

   } else {

     vm_exit_during_initialization("G1 requires a mod ref bs.");

     return JNI_ENOMEM;

   }

   // Also create a G1 rem set.

   if (G1UseHRIntoRS) {

     if (mr_bs()->is_a(BarrierSet::CardTableModRef)) {

       _g1_rem_set = new HRInto_G1RemSet(this, (CardTableModRefBS*)mr_bs());

     } else {

       vm_exit_during_initialization("G1 requires a cardtable mod ref bs.");

       return JNI_ENOMEM;

     }

   } else {

     _g1_rem_set = new StupidG1RemSet(this);

   }

   // Carve out the G1 part of the heap.

   ReservedSpace g1_rs   = heap_rs.first_part(max_byte_size);

   _g1_reserved = MemRegion((HeapWord*)g1_rs.base(),

                            g1_rs.size()/HeapWordSize);

   ReservedSpace perm_gen_rs = heap_rs.last_part(max_byte_size);

   _perm_gen = pgs->init(perm_gen_rs, pgs->init_size(), rem_set());

   _g1_storage.initialize(g1_rs, 0);

   _g1_committed = MemRegion((HeapWord*)_g1_storage.low(), (size_t) 0);

   _g1_max_committed = _g1_committed;

   _hrs = new HeapRegionSeq(_expansion_regions);

   guarantee(_hrs != NULL, "Couldn't allocate HeapRegionSeq");

   guarantee(_cur_alloc_region == NULL, "from constructor");

   _bot_shared = new G1BlockOffsetSharedArray(_reserved,

                                              heap_word_size(init_byte_size));

   _g1h = this;

   // Create the ConcurrentMark data structure and thread.

   // (Must do this late, so that "max_regions" is defined.)

   _cm       = new ConcurrentMark(heap_rs, (int) max_regions());

   _cmThread = _cm->cmThread();

   // ...and the concurrent zero-fill thread, if necessary.

   if (G1ConcZeroFill) {

     _czft = new ConcurrentZFThread();

   }

   // Allocate the popular regions; take them off free lists.

   size_t pop_byte_size = G1NumPopularRegions * HeapRegion::GrainBytes;

   expand(pop_byte_size);

   _popular_object_boundary =

     _g1_reserved.start() + (G1NumPopularRegions * HeapRegion::GrainWords);

   for (int i = 0; i < G1NumPopularRegions; i++) {

     HeapRegion* hr = newAllocRegion(HeapRegion::GrainWords);

     //    assert(hr != NULL && hr->bottom() < _popular_object_boundary,

     //     "Should be enough, and all should be below boundary.");

     hr->set_popular(true);

   }

   assert(_cur_pop_hr_index == 0, "Start allocating at the first region.");

   // Initialize the from_card cache structure of HeapRegionRemSet.

   HeapRegionRemSet::init_heap(max_regions());

   // Now expand into the rest of the initial heap size.

   expand(init_byte_size - pop_byte_size);

   // Perform any initialization actions delegated to the policy.

   g1_policy()->init();

   g1_policy()->note_start_of_mark_thread();

   _refine_cte_cl =

     new RefineCardTableEntryClosure(ConcurrentG1RefineThread::sts(),

                                     g1_rem_set(),

                                     concurrent_g1_refine());

   JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

   JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,

                                                SATB_Q_FL_lock,

                                                0,

                                                Shared_SATB_Q_lock);

   if (G1RSBarrierUseQueue) {

     JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,

                                                   DirtyCardQ_FL_lock,

                                                   G1DirtyCardQueueMax,

                                                   Shared_DirtyCardQ_lock);

   }

   if (G1DeferredRSUpdate) {

     dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,

                                       DirtyCardQ_FL_lock,

                                       0,

                                       Shared_DirtyCardQ_lock,

                                       &JavaThread::dirty_card_queue_set());

   }

   // In case we're keeping closure specialization stats, initialize those

   // counts and that mechanism.

   SpecializationStats::clear();

   _gc_alloc_region_list = NULL;

   // Do later initialization work for concurrent refinement.

   _cg1r->init();

   const char* group_names[] = { "CR", "ZF", "CM", "CL" };

   GCOverheadReporter::initGCOverheadReporter(4, group_names);

   return JNI_OK;

 }

 void G1CollectedHeap::ref_processing_init() {

   SharedHeap::ref_processing_init();

   MemRegion mr = reserved_region();

   _ref_processor = ReferenceProcessor::create_ref_processor(

                                          mr,    // span

                                          false, // Reference discovery is not atomic

                                                 // (though it shouldn't matter here.)

                                          true,  // mt_discovery

                                          NULL,  // is alive closure: need to fill this in for efficiency

                                          ParallelGCThreads,

                                          ParallelRefProcEnabled,

                                          true); // Setting next fields of discovered

                                                 // lists requires a barrier.

 }

 size_t G1CollectedHeap::capacity() const {

   return _g1_committed.byte_size();

 }

 void G1CollectedHeap::iterate_dirty_card_closure(bool concurrent,

                                                  int worker_i) {

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   int n_completed_buffers = 0;

   while (dcqs.apply_closure_to_completed_buffer(worker_i, 0, true)) {

     n_completed_buffers++;

   }

   g1_policy()->record_update_rs_processed_buffers(worker_i,

                                                   (double) n_completed_buffers);

   dcqs.clear_n_completed_buffers();

   // Finish up the queue...

   if (worker_i == 0) concurrent_g1_refine()->clean_up_cache(worker_i,

                                                             g1_rem_set());

   assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!");

 }

 // Computes the sum of the storage used by the various regions.

 size_t G1CollectedHeap::used() const {

   assert(Heap_lock->owner() != NULL,

          "Should be owned on this thread's behalf.");

   size_t result = _summary_bytes_used;

   if (_cur_alloc_region != NULL)

     result += _cur_alloc_region->used();

   return result;

 }

 class SumUsedClosure: public HeapRegionClosure {

   size_t _used;

 public:

   SumUsedClosure() : _used(0) {}

   bool doHeapRegion(HeapRegion* r) {

     if (!r->continuesHumongous()) {

       _used += r->used();

     }

     return false;

   }

   size_t result() { return _used; }

 };

 size_t G1CollectedHeap::recalculate_used() const {

   SumUsedClosure blk;

   _hrs->iterate(&blk);

   return blk.result();

 }

 #ifndef PRODUCT

 class SumUsedRegionsClosure: public HeapRegionClosure {

   size_t _num;

 public:

   // _num is set to 1 to account for the popular region

   SumUsedRegionsClosure() : _num(G1NumPopularRegions) {}

   bool doHeapRegion(HeapRegion* r) {

     if (r->continuesHumongous() || r->used() > 0 || r->is_gc_alloc_region()) {

       _num += 1;

     }

     return false;

   }

   size_t result() { return _num; }

 };

 size_t G1CollectedHeap::recalculate_used_regions() const {

   SumUsedRegionsClosure blk;

   _hrs->iterate(&blk);

   return blk.result();

 }

 #endif // PRODUCT

 size_t G1CollectedHeap::unsafe_max_alloc() {

   if (_free_regions > 0) return HeapRegion::GrainBytes;

   // otherwise, is there space in the current allocation region?

   // We need to store the current allocation region in a local variable

   // here. The problem is that this method doesn't take any locks and

   // there may be other threads which overwrite the current allocation

   // region field. attempt_allocation(), for example, sets it to NULL

   // and this can happen *after* the NULL check here but before the call

   // to free(), resulting in a SIGSEGV. Note that this doesn't appear

   // to be a problem in the optimized build, since the two loads of the

   // current allocation region field are optimized away.

   HeapRegion* car = _cur_alloc_region;

   // FIXME: should iterate over all regions?

   if (car == NULL) {

     return 0;

   }

   return car->free();

 }

 void G1CollectedHeap::collect(GCCause::Cause cause) {

   // The caller doesn't have the Heap_lock

   assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock");

   MutexLocker ml(Heap_lock);

   collect_locked(cause);

 }

 void G1CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) {

   assert(Thread::current()->is_VM_thread(), "Precondition#1");

   assert(Heap_lock->is_locked(), "Precondition#2");

   GCCauseSetter gcs(this, cause);

   switch (cause) {

     case GCCause::_heap_inspection:

     case GCCause::_heap_dump: {

       HandleMark hm;

       do_full_collection(false);         // don't clear all soft refs

       break;

     }

     default: // XXX FIX ME

       ShouldNotReachHere(); // Unexpected use of this function

   }

 }

 void G1CollectedHeap::collect_locked(GCCause::Cause cause) {

   // Don't want to do a GC until cleanup is completed.

   wait_for_cleanup_complete();

   // Read the GC count while holding the Heap_lock

   int gc_count_before = SharedHeap::heap()->total_collections();

   {

     MutexUnlocker mu(Heap_lock);  // give up heap lock, execute gets it back

     VM_G1CollectFull op(gc_count_before, cause);

     VMThread::execute(&op);

   }

 }

 bool G1CollectedHeap::is_in(const void* p) const {

   if (_g1_committed.contains(p)) {

     HeapRegion* hr = _hrs->addr_to_region(p);

     return hr->is_in(p);

   } else {

     return _perm_gen->as_gen()->is_in(p);

   }

 }

 // Iteration functions.

 // Iterates an OopClosure over all ref-containing fields of objects

 // within a HeapRegion.

 class IterateOopClosureRegionClosure: public HeapRegionClosure {

   MemRegion _mr;

   OopClosure* _cl;

 public:

   IterateOopClosureRegionClosure(MemRegion mr, OopClosure* cl)

     : _mr(mr), _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     if (! r->continuesHumongous()) {

       r->oop_iterate(_cl);

     }

     return false;

   }

 };

 void G1CollectedHeap::oop_iterate(OopClosure* cl) {

   IterateOopClosureRegionClosure blk(_g1_committed, cl);

   _hrs->iterate(&blk);

 }

 void G1CollectedHeap::oop_iterate(MemRegion mr, OopClosure* cl) {

   IterateOopClosureRegionClosure blk(mr, cl);

   _hrs->iterate(&blk);

 }

 // Iterates an ObjectClosure over all objects within a HeapRegion.

 class IterateObjectClosureRegionClosure: public HeapRegionClosure {

   ObjectClosure* _cl;

 public:

   IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     if (! r->continuesHumongous()) {

       r->object_iterate(_cl);

     }

     return false;

   }

 };

 void G1CollectedHeap::object_iterate(ObjectClosure* cl) {

   IterateObjectClosureRegionClosure blk(cl);

   _hrs->iterate(&blk);

 }

 void G1CollectedHeap::object_iterate_since_last_GC(ObjectClosure* cl) {

   // FIXME: is this right?

   guarantee(false, "object_iterate_since_last_GC not supported by G1 heap");

 }

 // Calls a SpaceClosure on a HeapRegion.

 class SpaceClosureRegionClosure: public HeapRegionClosure {

   SpaceClosure* _cl;

 public:

   SpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {}

   bool doHeapRegion(HeapRegion* r) {

     _cl->do_space(r);

     return false;

   }

 };

 void G1CollectedHeap::space_iterate(SpaceClosure* cl) {

   SpaceClosureRegionClosure blk(cl);

   _hrs->iterate(&blk);

 }

 void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) {

   _hrs->iterate(cl);

 }

 void G1CollectedHeap::heap_region_iterate_from(HeapRegion* r,

                                                HeapRegionClosure* cl) {

   _hrs->iterate_from(r, cl);

 }

 void

 G1CollectedHeap::heap_region_iterate_from(int idx, HeapRegionClosure* cl) {

   _hrs->iterate_from(idx, cl);

 }

 HeapRegion* G1CollectedHeap::region_at(size_t idx) { return _hrs->at(idx); }

 void

 G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl,

                                                  int worker,

                                                  jint claim_value) {

   const size_t regions = n_regions();

   const size_t worker_num = (ParallelGCThreads > 0 ? ParallelGCThreads : 1);

   // try to spread out the starting points of the workers

   const size_t start_index = regions / worker_num * (size_t) worker;

   // each worker will actually look at all regions

   for (size_t count = 0; count < regions; ++count) {

     const size_t index = (start_index + count) % regions;

     assert(0 <= index && index < regions, "sanity");

     HeapRegion* r = region_at(index);

     // we'll ignore "continues humongous" regions (we'll process them

     // when we come across their corresponding "start humongous"

     // region) and regions already claimed

     if (r->claim_value() == claim_value || r->continuesHumongous()) {

       continue;

     }

     // OK, try to claim it

     if (r->claimHeapRegion(claim_value)) {

       // success!

       assert(!r->continuesHumongous(), "sanity");

       if (r->startsHumongous()) {

         // If the region is "starts humongous" we'll iterate over its

         // "continues humongous" first; in fact we'll do them

         // first. The order is important. In on case, calling the

         // closure on the "starts humongous" region might de-allocate

         // and clear all its "continues humongous" regions and, as a

         // result, we might end up processing them twice. So, we'll do

         // them first (notice: most closures will ignore them anyway) and

         // then we'll do the "starts humongous" region.

         for (size_t ch_index = index + 1; ch_index < regions; ++ch_index) {

           HeapRegion* chr = region_at(ch_index);

           // if the region has already been claimed or it's not

           // "continues humongous" we're done

           if (chr->claim_value() == claim_value ||

               !chr->continuesHumongous()) {

             break;

           }

           // Noone should have claimed it directly. We can given

           // that we claimed its "starts humongous" region.

           assert(chr->claim_value() != claim_value, "sanity");

           assert(chr->humongous_start_region() == r, "sanity");

           if (chr->claimHeapRegion(claim_value)) {

             // we should always be able to claim it; noone else should

             // be trying to claim this region

             bool res2 = cl->doHeapRegion(chr);

             assert(!res2, "Should not abort");

             // Right now, this holds (i.e., no closure that actually

             // does something with "continues humongous" regions

             // clears them). We might have to weaken it in the future,

             // but let's leave these two asserts here for extra safety.

             assert(chr->continuesHumongous(), "should still be the case");

             assert(chr->humongous_start_region() == r, "sanity");

           } else {

             guarantee(false, "we should not reach here");

           }

         }

       }

       assert(!r->continuesHumongous(), "sanity");

       bool res = cl->doHeapRegion(r);

       assert(!res, "Should not abort");

     }

   }

 }

 class ResetClaimValuesClosure: public HeapRegionClosure {

 public:

   bool doHeapRegion(HeapRegion* r) {

     r->set_claim_value(HeapRegion::InitialClaimValue);

     return false;

   }

 };

 void

 G1CollectedHeap::reset_heap_region_claim_values() {

   ResetClaimValuesClosure blk;

   heap_region_iterate(&blk);

 }

 #ifdef ASSERT

 // This checks whether all regions in the heap have the correct claim

 // value. I also piggy-backed on this a check to ensure that the

 // humongous_start_region() information on "continues humongous"

 // regions is correct.

 class CheckClaimValuesClosure : public HeapRegionClosure {

 private:

   jint _claim_value;

   size_t _failures;

   HeapRegion* _sh_region;

 public:

   CheckClaimValuesClosure(jint claim_value) :

     _claim_value(claim_value), _failures(0), _sh_region(NULL) { }

   bool doHeapRegion(HeapRegion* r) {

     if (r->claim_value() != _claim_value) {

       gclog_or_tty->print_cr("Region ["PTR_FORMAT","PTR_FORMAT"), "

                              "claim value = %d, should be %d",

                              r->bottom(), r->end(), r->claim_value(),

                              _claim_value);

       ++_failures;

     }

     if (!r->isHumongous()) {

       _sh_region = NULL;

     } else if (r->startsHumongous()) {

       _sh_region = r;

     } else if (r->continuesHumongous()) {

       if (r->humongous_start_region() != _sh_region) {

         gclog_or_tty->print_cr("Region ["PTR_FORMAT","PTR_FORMAT"), "

                                "HS = "PTR_FORMAT", should be "PTR_FORMAT,

                                r->bottom(), r->end(),

                                r->humongous_start_region(),

                                _sh_region);

         ++_failures;

       }

     }

     return false;

   }

   size_t failures() {

     return _failures;

   }

 };

 bool G1CollectedHeap::check_heap_region_claim_values(jint claim_value) {

   CheckClaimValuesClosure cl(claim_value);

   heap_region_iterate(&cl);

   return cl.failures() == 0;

 }

 #endif // ASSERT

 void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) {

   HeapRegion* r = g1_policy()->collection_set();

   while (r != NULL) {

     HeapRegion* next = r->next_in_collection_set();

     if (cl->doHeapRegion(r)) {

       cl->incomplete();

       return;

     }

     r = next;

   }

 }

 void G1CollectedHeap::collection_set_iterate_from(HeapRegion* r,

                                                   HeapRegionClosure *cl) {

   assert(r->in_collection_set(),

          "Start region must be a member of the collection set.");

   HeapRegion* cur = r;

   while (cur != NULL) {

     HeapRegion* next = cur->next_in_collection_set();

     if (cl->doHeapRegion(cur) && false) {

       cl->incomplete();

       return;

     }

     cur = next;

   }

   cur = g1_policy()->collection_set();

   while (cur != r) {

     HeapRegion* next = cur->next_in_collection_set();

     if (cl->doHeapRegion(cur) && false) {

       cl->incomplete();

       return;

     }

     cur = next;

   }

 }

 CompactibleSpace* G1CollectedHeap::first_compactible_space() {

   return _hrs->length() > 0 ? _hrs->at(0) : NULL;

 }

 Space* G1CollectedHeap::space_containing(const void* addr) const {

   Space* res = heap_region_containing(addr);

   if (res == NULL)

     res = perm_gen()->space_containing(addr);

   return res;

 }

 HeapWord* G1CollectedHeap::block_start(const void* addr) const {

   Space* sp = space_containing(addr);

   if (sp != NULL) {

     return sp->block_start(addr);

   }

   return NULL;

 }

 size_t G1CollectedHeap::block_size(const HeapWord* addr) const {

   Space* sp = space_containing(addr);

   assert(sp != NULL, "block_size of address outside of heap");

   return sp->block_size(addr);

 }

 bool G1CollectedHeap::block_is_obj(const HeapWord* addr) const {

   Space* sp = space_containing(addr);

   return sp->block_is_obj(addr);

 }

 bool G1CollectedHeap::supports_tlab_allocation() const {

   return true;

 }

 size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const {

   return HeapRegion::GrainBytes;

 }

 size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const {

   // Return the remaining space in the cur alloc region, but not less than

   // the min TLAB size.

   // Also, no more than half the region size, since we can't allow tlabs to

   // grow big enough to accomodate humongous objects.

   // We need to story it locally, since it might change between when we

   // test for NULL and when we use it later.

   ContiguousSpace* cur_alloc_space = _cur_alloc_region;

   if (cur_alloc_space == NULL) {

     return HeapRegion::GrainBytes/2;

   } else {

     return MAX2(MIN2(cur_alloc_space->free(),

                      (size_t)(HeapRegion::GrainBytes/2)),

                 (size_t)MinTLABSize);

   }

 }

 HeapWord* G1CollectedHeap::allocate_new_tlab(size_t size) {

   bool dummy;

   return G1CollectedHeap::mem_allocate(size, false, true, &dummy);

 }

 bool G1CollectedHeap::allocs_are_zero_filled() {

   return false;

 }

 size_t G1CollectedHeap::large_typearray_limit() {

   // FIXME

   return HeapRegion::GrainBytes/HeapWordSize;

 }

 size_t G1CollectedHeap::max_capacity() const {

   return _g1_committed.byte_size();

 }

 jlong G1CollectedHeap::millis_since_last_gc() {

   // assert(false, "NYI");

   return 0;

 }

 void G1CollectedHeap::prepare_for_verify() {

   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {

     ensure_parsability(false);

   }

   g1_rem_set()->prepare_for_verify();

 }

 class VerifyLivenessOopClosure: public OopClosure {

   G1CollectedHeap* g1h;

 public:

   VerifyLivenessOopClosure(G1CollectedHeap* _g1h) {

     g1h = _g1h;

   }

   void do_oop(narrowOop *p) {

     guarantee(false, "NYI");

   }

   void do_oop(oop *p) {

     oop obj = *p;

     assert(obj == NULL || !g1h->is_obj_dead(obj),

            "Dead object referenced by a not dead object");

   }

 };

 class VerifyObjsInRegionClosure: public ObjectClosure {

   G1CollectedHeap* _g1h;

   size_t _live_bytes;

   HeapRegion *_hr;

 public:

   VerifyObjsInRegionClosure(HeapRegion *hr) : _live_bytes(0), _hr(hr) {

     _g1h = G1CollectedHeap::heap();

   }

   void do_object(oop o) {

     VerifyLivenessOopClosure isLive(_g1h);

     assert(o != NULL, "Huh?");

     if (!_g1h->is_obj_dead(o)) {

       o->oop_iterate(&isLive);

       if (!_hr->obj_allocated_since_prev_marking(o))

         _live_bytes += (o->size() * HeapWordSize);

     }

   }

   size_t live_bytes() { return _live_bytes; }

 };

 class PrintObjsInRegionClosure : public ObjectClosure {

   HeapRegion *_hr;

   G1CollectedHeap *_g1;

 public:

   PrintObjsInRegionClosure(HeapRegion *hr) : _hr(hr) {

     _g1 = G1CollectedHeap::heap();

   };

   void do_object(oop o) {

     if (o != NULL) {

       HeapWord *start = (HeapWord *) o;

       size_t word_sz = o->size();

       gclog_or_tty->print("\nPrinting obj "PTR_FORMAT" of size " SIZE_FORMAT

                           " isMarkedPrev %d isMarkedNext %d isAllocSince %d\n",

                           (void*) o, word_sz,

                           _g1->isMarkedPrev(o),

                           _g1->isMarkedNext(o),

                           _hr->obj_allocated_since_prev_marking(o));

       HeapWord *end = start + word_sz;

       HeapWord *cur;

       int *val;

       for (cur = start; cur < end; cur++) {

         val = (int *) cur;

         gclog_or_tty->print("\t "PTR_FORMAT":"PTR_FORMAT"\n", val, *val);

       }

     }

   }

 };

 class VerifyRegionClosure: public HeapRegionClosure {

 public:

   bool _allow_dirty;

   bool _par;

   VerifyRegionClosure(bool allow_dirty, bool par = false)

     : _allow_dirty(allow_dirty), _par(par) {}

   bool doHeapRegion(HeapRegion* r) {

     guarantee(_par || r->claim_value() == HeapRegion::InitialClaimValue,

               "Should be unclaimed at verify points.");

     if (r->isHumongous()) {

       if (r->startsHumongous()) {

         // Verify the single H object.

         oop(r->bottom())->verify();

         size_t word_sz = oop(r->bottom())->size();

         guarantee(r->top() == r->bottom() + word_sz,

                   "Only one object in a humongous region");

       }

     } else {

       VerifyObjsInRegionClosure not_dead_yet_cl(r);

       r->verify(_allow_dirty);

       r->object_iterate(&not_dead_yet_cl);

       guarantee(r->max_live_bytes() >= not_dead_yet_cl.live_bytes(),

                 "More live objects than counted in last complete marking.");

     }

     return false;

   }

 };

 class VerifyRootsClosure: public OopsInGenClosure {

 private:

   G1CollectedHeap* _g1h;

   bool             _failures;

 public:

   VerifyRootsClosure() :

     _g1h(G1CollectedHeap::heap()), _failures(false) { }

   bool failures() { return _failures; }

   void do_oop(narrowOop* p) {

     guarantee(false, "NYI");

   }

   void do_oop(oop* p) {

     oop obj = *p;

     if (obj != NULL) {

       if (_g1h->is_obj_dead(obj)) {

         gclog_or_tty->print_cr("Root location "PTR_FORMAT" "

                                "points to dead obj "PTR_FORMAT, p, (void*) obj);

         obj->print_on(gclog_or_tty);

         _failures = true;

       }

     }

   }

 };

 // This is the task used for parallel heap verification.

 class G1ParVerifyTask: public AbstractGangTask {

 private:

   G1CollectedHeap* _g1h;

   bool _allow_dirty;

 public:

   G1ParVerifyTask(G1CollectedHeap* g1h, bool allow_dirty) :

     AbstractGangTask("Parallel verify task"),

     _g1h(g1h), _allow_dirty(allow_dirty) { }

   void work(int worker_i) {

     VerifyRegionClosure blk(_allow_dirty, true);

     _g1h->heap_region_par_iterate_chunked(&blk, worker_i,

                                           HeapRegion::ParVerifyClaimValue);

   }

 };

 void G1CollectedHeap::verify(bool allow_dirty, bool silent) {

   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {

     if (!silent) { gclog_or_tty->print("roots "); }

     VerifyRootsClosure rootsCl;

     process_strong_roots(false,

                          SharedHeap::SO_AllClasses,

                          &rootsCl,

                          &rootsCl);

     rem_set()->invalidate(perm_gen()->used_region(), false);

     if (!silent) { gclog_or_tty->print("heapRegions "); }

     if (GCParallelVerificationEnabled && ParallelGCThreads > 1) {

       assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue),

              "sanity check");

       G1ParVerifyTask task(this, allow_dirty);

       int n_workers = workers()->total_workers();

       set_par_threads(n_workers);

       workers()->run_task(&task);

       set_par_threads(0);

       assert(check_heap_region_claim_values(HeapRegion::ParVerifyClaimValue),

              "sanity check");

       reset_heap_region_claim_values();

       assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue),

              "sanity check");

     } else {

       VerifyRegionClosure blk(allow_dirty);

       _hrs->iterate(&blk);

     }

     if (!silent) gclog_or_tty->print("remset ");

     rem_set()->verify();

     guarantee(!rootsCl.failures(), "should not have had failures");

   } else {

     if (!silent) gclog_or_tty->print("(SKIPPING roots, heapRegions, remset) ");

   }

 }

 class PrintRegionClosure: public HeapRegionClosure {

   outputStream* _st;

 public:

   PrintRegionClosure(outputStream* st) : _st(st) {}

   bool doHeapRegion(HeapRegion* r) {

     r->print_on(_st);

     return false;

   }

 };

 void G1CollectedHeap::print() const { print_on(gclog_or_tty); }

 void G1CollectedHeap::print_on(outputStream* st) const {

   PrintRegionClosure blk(st);

   _hrs->iterate(&blk);

 }

 void G1CollectedHeap::print_gc_threads_on(outputStream* st) const {

   if (ParallelGCThreads > 0) {

     workers()->print_worker_threads();

   }

   st->print("\"G1 concurrent mark GC Thread\" ");

   _cmThread->print();

   st->cr();

   st->print("\"G1 concurrent refinement GC Thread\" ");

   _cg1r->cg1rThread()->print_on(st);

   st->cr();

   st->print("\"G1 zero-fill GC Thread\" ");

   _czft->print_on(st);

   st->cr();

 }

 void G1CollectedHeap::gc_threads_do(ThreadClosure* tc) const {

   if (ParallelGCThreads > 0) {

     workers()->threads_do(tc);

   }

   tc->do_thread(_cmThread);

   tc->do_thread(_cg1r->cg1rThread());

   tc->do_thread(_czft);

 }

 void G1CollectedHeap::print_tracing_info() const {

   concurrent_g1_refine()->print_final_card_counts();

   // We'll overload this to mean "trace GC pause statistics."

   if (TraceGen0Time || TraceGen1Time) {

     // The "G1CollectorPolicy" is keeping track of these stats, so delegate

     // to that.

     g1_policy()->print_tracing_info();

   }

   if (SummarizeG1RSStats) {

     g1_rem_set()->print_summary_info();

   }

   if (SummarizeG1ConcMark) {

     concurrent_mark()->print_summary_info();

   }

   if (SummarizeG1ZFStats) {

     ConcurrentZFThread::print_summary_info();

   }

   if (G1SummarizePopularity) {

     print_popularity_summary_info();

   }

   g1_policy()->print_yg_surv_rate_info();

   GCOverheadReporter::printGCOverhead();

   SpecializationStats::print();

 }

 int G1CollectedHeap::addr_to_arena_id(void* addr) const {

   HeapRegion* hr = heap_region_containing(addr);

   if (hr == NULL) {

     return 0;

   } else {

     return 1;

   }

 }

 G1CollectedHeap* G1CollectedHeap::heap() {

   assert(_sh->kind() == CollectedHeap::G1CollectedHeap,

          "not a garbage-first heap");

   return _g1h;

 }

 void G1CollectedHeap::gc_prologue(bool full /* Ignored */) {

   if (PrintHeapAtGC){

     gclog_or_tty->print_cr(" {Heap before GC collections=%d:", total_collections());

     Universe::print();

   }

   assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");

   // Call allocation profiler

   AllocationProfiler::iterate_since_last_gc();

   // Fill TLAB's and such

   ensure_parsability(true);

 }

 void G1CollectedHeap::gc_epilogue(bool full /* Ignored */) {

   // FIXME: what is this about?

   // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled"

   // is set.

   COMPILER2_PRESENT(assert(DerivedPointerTable::is_empty(),

                         "derived pointer present"));

   if (PrintHeapAtGC){

     gclog_or_tty->print_cr(" Heap after GC collections=%d:", total_collections());

     Universe::print();

     gclog_or_tty->print("} ");

   }

 }

 void G1CollectedHeap::do_collection_pause() {

   // Read the GC count while holding the Heap_lock

   // we need to do this _before_ wait_for_cleanup_complete(), to

   // ensure that we do not give up the heap lock and potentially

   // pick up the wrong count

   int gc_count_before = SharedHeap::heap()->total_collections();

   // Don't want to do a GC pause while cleanup is being completed!

   wait_for_cleanup_complete();

   g1_policy()->record_stop_world_start();

   {

     MutexUnlocker mu(Heap_lock);  // give up heap lock, execute gets it back

     VM_G1IncCollectionPause op(gc_count_before);

     VMThread::execute(&op);

   }

 }

 void

 G1CollectedHeap::doConcurrentMark() {

   if (G1ConcMark) {

     MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);

     if (!_cmThread->in_progress()) {

       _cmThread->set_started();

       CGC_lock->notify();

     }

   }

 }

 class VerifyMarkedObjsClosure: public ObjectClosure {

     G1CollectedHeap* _g1h;

     public:

     VerifyMarkedObjsClosure(G1CollectedHeap* g1h) : _g1h(g1h) {}

     void do_object(oop obj) {

       assert(obj->mark()->is_marked() ? !_g1h->is_obj_dead(obj) : true,

              "markandsweep mark should agree with concurrent deadness");

     }

 };

 void

 G1CollectedHeap::checkConcurrentMark() {

     VerifyMarkedObjsClosure verifycl(this);

     //    MutexLockerEx x(getMarkBitMapLock(),

     //              Mutex::_no_safepoint_check_flag);

     object_iterate(&verifycl);

 }

 void G1CollectedHeap::do_sync_mark() {

   _cm->checkpointRootsInitial();

   _cm->markFromRoots();

   _cm->checkpointRootsFinal(false);

 }

 // <NEW PREDICTION>

 double G1CollectedHeap::predict_region_elapsed_time_ms(HeapRegion *hr,

                                                        bool young) {

   return _g1_policy->predict_region_elapsed_time_ms(hr, young);

 }

 void G1CollectedHeap::check_if_region_is_too_expensive(double

                                                            predicted_time_ms) {

   _g1_policy->check_if_region_is_too_expensive(predicted_time_ms);

 }

 size_t G1CollectedHeap::pending_card_num() {

   size_t extra_cards = 0;

   JavaThread *curr = Threads::first();

   while (curr != NULL) {

     DirtyCardQueue& dcq = curr->dirty_card_queue();

     extra_cards += dcq.size();

     curr = curr->next();

   }

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   size_t buffer_size = dcqs.buffer_size();

   size_t buffer_num = dcqs.completed_buffers_num();

   return buffer_size * buffer_num + extra_cards;

 }

 size_t G1CollectedHeap::max_pending_card_num() {

   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

   size_t buffer_size = dcqs.buffer_size();

   size_t buffer_num  = dcqs.completed_buffers_num();

   int thread_num  = Threads::number_of_threads();

   return (buffer_num + thread_num) * buffer_size;

 }

 size_t G1CollectedHeap::cards_scanned() {

   HRInto_G1RemSet* g1_rset = (HRInto_G1RemSet*) g1_rem_set();

   return g1_rset->cardsScanned();

 }

 void

 G1CollectedHeap::setup_surviving_young_words() {

   guarantee( _surviving_young_words == NULL, "pre-condition" );

   size_t array_length = g1_policy()->young_cset_length();

   _surviving_young_words = NEW_C_HEAP_ARRAY(size_t, array_length);

   if (_surviving_young_words == NULL) {

     vm_exit_out_of_memory(sizeof(size_t) * array_length,

                           "Not enough space for young surv words summary.");

   }

   memset(_surviving_young_words, 0, array_length * sizeof(size_t));

   for (size_t i = 0;  i < array_length; ++i) {

     guarantee( _surviving_young_words[i] == 0, "invariant" );

   }

 }

 void

 G1CollectedHeap::update_surviving_young_words(size_t* surv_young_words) {

   MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);

   size_t array_length = g1_policy()->young_cset_length();

   for (size_t i = 0; i < array_length; ++i)

     _surviving_young_words[i] += surv_young_words[i];

 }

 void

 G1CollectedHeap::cleanup_surviving_young_words() {

   guarantee( _surviving_young_words != NULL, "pre-condition" );

   FREE_C_HEAP_ARRAY(size_t, _surviving_young_words);

   _surviving_young_words = NULL;

 }

 // </NEW PREDICTION>

 void

 G1CollectedHeap::do_collection_pause_at_safepoint(HeapRegion* popular_region) {

   char verbose_str[128];

   sprintf(verbose_str, "GC pause ");

   if (popular_region != NULL)

     strcat(verbose_str, "(popular)");

   else if (g1_policy()->in_young_gc_mode()) {

     if (g1_policy()->full_young_gcs())

       strcat(verbose_str, "(young)");

     else

       strcat(verbose_str, "(partial)");

   }

   bool reset_should_initiate_conc_mark = false;

   if (popular_region != NULL && g1_policy()->should_initiate_conc_mark()) {

     // we currently do not allow an initial mark phase to be piggy-backed

     // on a popular pause

     reset_should_initiate_conc_mark = true;

     g1_policy()->unset_should_initiate_conc_mark();

   }

   if (g1_policy()->should_initiate_conc_mark())

     strcat(verbose_str, " (initial-mark)");

   GCCauseSetter x(this, (popular_region == NULL ?

                          GCCause::_g1_inc_collection_pause :

                          GCCause::_g1_pop_region_collection_pause));

   // if PrintGCDetails is on, we'll print long statistics information

   // in the collector policy code, so let's not print this as the output

   // is messy if we do.

   gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);

   TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);

   TraceTime t(verbose_str, PrintGC && !PrintGCDetails, true, gclog_or_tty);

   ResourceMark rm;

   assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");

   assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread");

   guarantee(!is_gc_active(), "collection is not reentrant");

   assert(regions_accounted_for(), "Region leakage!");

   increment_gc_time_stamp();

   if (g1_policy()->in_young_gc_mode()) {

     assert(check_young_list_well_formed(),

                 "young list should be well formed");

   }

   if (GC_locker::is_active()) {

     return; // GC is disabled (e.g. JNI GetXXXCritical operation)

   }

   bool abandoned = false;

   { // Call to jvmpi::post_class_unload_events must occur outside of active GC

     IsGCActiveMark x;

     gc_prologue(false);

     increment_total_collections();

 #if G1_REM_SET_LOGGING

     gclog_or_tty->print_cr("\nJust chose CS, heap:");

     print();

 #endif

     if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {

       HandleMark hm;  // Discard invalid handles created during verification

       prepare_for_verify();

       gclog_or_tty->print(" VerifyBeforeGC:");

       Universe::verify(false);

     }

     COMPILER2_PRESENT(DerivedPointerTable::clear());

     // We want to turn off ref discovery, if necessary, and turn it back on

     // on again later if we do.

     bool was_enabled = ref_processor()->discovery_enabled();

     if (was_enabled) ref_processor()->disable_discovery();

     // Forget the current alloc region (we might even choose it to be part

     // of the collection set!).

     abandon_cur_alloc_region();

     // The elapsed time induced by the start time below deliberately elides

     // the possible verification above.

     double start_time_sec = os::elapsedTime();

     GCOverheadReporter::recordSTWStart(start_time_sec);

     size_t start_used_bytes = used();

     if (!G1ConcMark) {

       do_sync_mark();

     }

     g1_policy()->record_collection_pause_start(start_time_sec,

                                                start_used_bytes);

     guarantee(_in_cset_fast_test == NULL, "invariant");

     guarantee(_in_cset_fast_test_base == NULL, "invariant");

     _in_cset_fast_test_length = max_regions();

     _in_cset_fast_test_base =

                              NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);

     memset(_in_cset_fast_test_base, false,

                                      _in_cset_fast_test_length * sizeof(bool));

     // We're biasing _in_cset_fast_test to avoid subtracting the

     // beginning of the heap every time we want to index; basically

     // it's the same with what we do with the card table.

     _in_cset_fast_test = _in_cset_fast_test_base -

               ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);

 #if SCAN_ONLY_VERBOSE

     _young_list->print();

 #endif // SCAN_ONLY_VERBOSE

     if (g1_policy()->should_initiate_conc_mark()) {

       concurrent_mark()->checkpointRootsInitialPre();

     }

     save_marks();

     // We must do this before any possible evacuation that should propagate

     // marks, including evacuation of popular objects in a popular pause.

     if (mark_in_progress()) {

       double start_time_sec = os::elapsedTime();

       _cm->drainAllSATBBuffers();

       double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0;

       g1_policy()->record_satb_drain_time(finish_mark_ms);

     }

     // Record the number of elements currently on the mark stack, so we

     // only iterate over these.  (Since evacuation may add to the mark

     // stack, doing more exposes race conditions.)  If no mark is in

     // progress, this will be zero.

     _cm->set_oops_do_bound();

     assert(regions_accounted_for(), "Region leakage.");

     bool abandoned = false;

     if (mark_in_progress())

       concurrent_mark()->newCSet();

     // Now choose the CS.

     if (popular_region == NULL) {

       g1_policy()->choose_collection_set();

     } else {

       // We may be evacuating a single region (for popularity).

       g1_policy()->record_popular_pause_preamble_start();

       popularity_pause_preamble(popular_region);

       g1_policy()->record_popular_pause_preamble_end();

       abandoned = (g1_policy()->collection_set() == NULL);

       // Now we allow more regions to be added (we have to collect

       // all popular regions).

       if (!abandoned) {

         g1_policy()->choose_collection_set(popular_region);

       }

     }

     // We may abandon a pause if we find no region that will fit in the MMU

     // pause.

     abandoned = (g1_policy()->collection_set() == NULL);

     // Nothing to do if we were unable to choose a collection set.

     if (!abandoned) {

 #if G1_REM_SET_LOGGING

       gclog_or_tty->print_cr("\nAfter pause, heap:");

       print();

 #endif

       setup_surviving_young_words();

       // Set up the gc allocation regions.

       get_gc_alloc_regions();

       // Actually do the work...

       evacuate_collection_set();

       free_collection_set(g1_policy()->collection_set());

       g1_policy()->clear_collection_set();

       FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base);

       // this is more for peace of mind; we're nulling them here and

       // we're expecting them to be null at the beginning of the next GC

       _in_cset_fast_test = NULL;

       _in_cset_fast_test_base = NULL;

       if (popular_region != NULL) {

         // We have to wait until now, because we don't want the region to

         // be rescheduled for pop-evac during RS update.

         popular_region->set_popular_pending(false);